소스 검색

feat: restructure of decoder and bridge

chore/restructure-decoder
Blaz Smehov 1 주 전
부모
커밋
ba93c59a8a
11개의 변경된 파일70523개의 추가작업 그리고 3920개의 파일을 삭제
  1. +70299
    -0
      cmd/decoder/analysis.txt
  2. +32
    -0
      cmd/decoder/examples.txt
  3. +144
    -51
      cmd/decoder/main.go
  4. +0
    -3488
      cmd/decoder/save.txt
  5. +0
    -150
      cmd/location/main.go
  6. +0
    -0
      cmd/presenSe/.keep
  7. +0
    -188
      cmd/presenSe/presense.go
  8. +8
    -8
      cmd/server/main.go
  9. +16
    -0
      internal/pkg/model/typeMethods.go
  10. +23
    -8
      internal/pkg/model/types.go
  11. +1
    -27
      scripts/testAPI.sh

+ 70299
- 0
cmd/decoder/analysis.txt
파일 크기가 너무 크기때문에 변경 상태를 표시하지 않습니다.
파일 보기


+ 32
- 0
cmd/decoder/examples.txt 파일 보기

@@ -0,0 +1,32 @@
Ingics iBS01G
id: C83F8F17DB35
020106 12 FF590080BC240100FFFFFFFF000000000000
id: C83F8F17DB35
020106 12 FF590080BC240100FFFFFFFF000000000000


type 0x16 - service Data

Minew B7
id: C300003947C4
020106 0303E1FF 1216E1FFA1031AFFFEFEFB0000C447390000C3
id: C300003947C4
0201061AFF4C000215FDA50693A4E24FB1AFCFC6EB0764782500000000EC - iBeacon
id: C300003947C4
020106 0303AAFE 1516AAFE00E800112233445566778899ABCDE7280002 - eddystone
id: C300003947C4
0201060303E1FF1216E1FFA1031AFFFEFEFB0000C447390000C3
id: C300003947C4
0201060303E1FF0E16E1FFA1081AC447390000C34237

Minew MWB01
id: C7AE561E38B7
02010617FF0001000000000000000000005F0700006F4C0000640003095336
id: C7AE561E38B7
02010617FF00020000FF0000FF0000FF0006001D005200000B200803095336

Minew MWC01
id: E01F9A7A47D2
02010617FF00020000FF0000FF0000FF0006001D005200000A242D03095332
id: E01F9A7A47D2
02010617FF000100000000000000000000780700006F4C0000640003095332

+ 144
- 51
cmd/decoder/main.go 파일 보기

@@ -1,16 +1,22 @@
package main

import (
"bytes"
"context"
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"math"
"strconv"
"strings"
"time"

"github.com/AFASystems/presence/internal/pkg/config"
"github.com/AFASystems/presence/internal/pkg/kafkaclient"
"github.com/AFASystems/presence/internal/pkg/model"
"github.com/AFASystems/presence/internal/pkg/mqttclient"
"github.com/segmentio/kafka-go"
)

func main() {
@@ -19,9 +25,6 @@ func main() {
Beacons: model.BeaconsList{
Beacons: make(map[string]model.Beacon),
},
LatestList: model.LatestBeaconsList{
LatestList: make(map[string]model.Beacon),
},
Settings: model.Settings{
Settings: model.SettingsVal{
Location_confidence: 4,
@@ -31,10 +34,12 @@ func main() {
HA_send_changes_only: false,
},
},
BeaconEvents: model.BeaconEventList{
Beacons: make(map[string]model.BeaconEvent),
},
BeaconsLookup: make(map[string]struct{}),
}

fmt.Println("init")

cfg := config.Load()

// Kafka reader for Raw MQTT beacons
@@ -42,8 +47,13 @@ func main() {
defer rawReader.Close()

// Kafka reader for API server updates
// apiReader := kafkaclient.KafkaReader(cfg.KafkaURL, "apibeacons", "gid-api")
// defer apiReader.Close()
apiReader := kafkaclient.KafkaReader(cfg.KafkaURL, "apibeacons", "gid-api")
defer apiReader.Close()

alertWriter := kafkaclient.KafkaWriter(cfg.KafkaURL, "alertbeacons")
defer alertWriter.Close()

fmt.Println("Decoder initialized, subscribed to Kafka topics")

// // Kafka reader for latest list updates
// latestReader := kafkaclient.KafkaReader(cfg.KafkaURL, "latestbeacons", "gid-latest")
@@ -55,36 +65,27 @@ func main() {

// declare channel for collecting Kafka messages
chRaw := make(chan model.Incoming_json, 2000)
// chApi := make(chan model.ApiUpdate, 2000)
chApi := make(chan model.ApiUpdate, 2000)
// chLatest := make(chan model.Incoming_json, 2000)
// chSettings := make(chan model.SettingsVal, 10)

go kafkaclient.Consume(rawReader, chRaw)
// go kafkaclient.Consume(apiReader, chApi)
go kafkaclient.Consume(apiReader, chApi)
// go kafkaclient.Consume(latestReader, chLatest)
// go kafkaclient.Consume(settingsReader, chSettings)

for {
select {
case msg := <-chRaw:
processIncoming(msg, &appCtx)
// case msg := <-chApi:
// switch msg.Method {
// case "POST":
// fmt.Println("Incoming POST")
// appCtx.Beacons.Lock.Lock()
// appCtx.Beacons.Beacons[msg.Beacon.Beacon_id] = msg.Beacon
// case "DELETE":
// fmt.Println("Incoming delete")
// _, exists := appCtx.Beacons.Beacons[msg.ID]
// if exists {
// appCtx.Beacons.Lock.Lock()
// delete(appCtx.Beacons.Beacons, msg.ID)
// }
// default:
// fmt.Println("unknown method: ", msg.Method)
// }
// appCtx.Beacons.Lock.Unlock()
processIncoming(msg, &appCtx, alertWriter)
case msg := <-chApi:
switch msg.Method {
case "POST":
id := msg.Beacon.Beacon_id
appCtx.BeaconsLookup[id] = struct{}{}
case "DELETE":
fmt.Println("Incoming delete message")
}
// case msg := <-chLatest:
// fmt.Println("latest msg: ", msg)
// case msg := <-chSettings:
@@ -96,44 +97,136 @@ func main() {
}
}

func processIncoming(incoming model.Incoming_json, ctx *model.AppContext) {
defer func() {
if err := recover(); err != nil {
fmt.Println("work failed:", err)
}
}()
func processIncoming(incoming model.Incoming_json, ctx *model.AppContext, writer *kafka.Writer) {
id := mqttclient.GetBeaconID(incoming)
_, ok := ctx.BeaconsLookup[id]
if !ok {
return
}

// Get ID
err := decodeBeacon(incoming, ctx, writer)
if err != nil {
fmt.Println("error in decoding")
return
}
}

func decodeBeacon(incoming model.Incoming_json, ctx *model.AppContext, writer *kafka.Writer) error {
beacon := strings.TrimSpace(incoming.Data)
id := mqttclient.GetBeaconID(incoming)
fmt.Println(incoming.Data)
if beacon == "" {
return nil // How to return error?, do I even need to return error
}

b, err := hex.DecodeString(beacon)
if err != nil {
return err
}

// check for flag byte, if first AD structure is flag bytes, remove it
if len(b) > 1 && b[1] == 0x01 {
l := int(b[0]) // length of AD structure
if 1+l <= len(b) {
b = b[1+l:]
}
}

adStructureIndeces := ParseADFast(b)
event := model.BeaconEvent{}
for _, r := range adStructureIndeces {
ad := b[r[0]:r[1]]
if checkIngics(ad) {
event = parseIngicsState(ad)
event.Id = id
event.Name = id
break
} else if checkEddystoneTLM(ad) {
event = parseEddystoneState(ad)
event.Id = id
event.Name = id
break
} else if checkMinewB7(ad) {
fmt.Println("Minew B7 vendor format")
break
}
}

beacons := &ctx.Beacons
if event.Id != "" {
prevEvent, ok := ctx.BeaconEvents.Beacons[id]
ctx.BeaconEvents.Beacons[id] = event
if ok && bytes.Equal(prevEvent.Hash(), event.Hash()) {
return nil
}

beacons.Lock.Lock()
defer beacons.Lock.Unlock()
eMsg, err := json.Marshal(event)
if err != nil {
return err
}

incoming = mqttclient.IncomingBeaconFilter(incoming)
err = writer.WriteMessages(context.Background(), kafka.Message{
Value: eMsg,
})

beacon, exists := beacons.Beacons[id]
if !exists {
return
if err != nil {
return err
}

fmt.Println("Message sent")
}

fmt.Printf("%+v\n", beacon)
return nil
}

updateBeacon(&beacon, incoming)
beacons.Beacons[id] = beacon
func checkIngics(ad []byte) bool {
if len(ad) >= 6 &&
ad[1] == 0xFF &&
ad[2] == 0x59 &&
ad[3] == 0x00 &&
ad[4] == 0x80 &&
ad[5] == 0xBC {
return true
}

return false
}

func parseIngicsState(ad []byte) model.BeaconEvent {
return model.BeaconEvent{
Battery: uint32(binary.LittleEndian.Uint16(ad[6:8])),
Event: int(ad[8]),
Type: "Ingics",
}
}

func checkEddystoneTLM(ad []byte) bool {
if len(ad) >= 4 &&
ad[1] == 0x16 &&
ad[2] == 0xAA &&
ad[3] == 0xFE &&
ad[4] == 0x20 {
return true
}

return false
}

func processBeacon(hexStr string) {
b, _ := hex.DecodeString(hexStr)
func parseEddystoneState(ad []byte) model.BeaconEvent {
return model.BeaconEvent{
Battery: uint32(binary.BigEndian.Uint16(ad[6:8])),
Type: "Eddystone",
}
}

if len(b) > 2 && b[0] == 0x02 && b[1] == 0x01 {
b = b[2+int(b[0]):]
// I dont think this is always true, but for testing is ok
func checkMinewB7(ad []byte) bool {
if len(ad) >= 4 &&
ad[1] == 0x16 &&
ad[2] == 0xE1 &&
ad[3] == 0xFF {
return true
}

ads := ParseADFast(b)
_ = ads
return false
}

func ParseADFast(b []byte) [][2]int {


+ 0
- 3488
cmd/decoder/save.txt
파일 크기가 너무 크기때문에 변경 상태를 표시하지 않습니다.
파일 보기


+ 0
- 150
cmd/location/main.go 파일 보기

@@ -1,150 +0,0 @@
package main

import (
"context"
"time"

"github.com/AFASystems/presence/internal/pkg/model"
presenseredis "github.com/AFASystems/presence/internal/pkg/redis"
"github.com/redis/go-redis/v9"
)

func main() {
ctx := context.Background()
client := redis.NewClient(&redis.Options{
Addr: "127.0.0.1:6379",
Password: "",
})
getLikelyLocations(client, ctx)
}

func getLikelyLocations(client *redis.Client, ctx context.Context) {
httpRes := model.HTTPResultsList{
HTTPResults: model.HTTPLocationsList{
Beacons: make([]model.HTTPLocation, 0),
},
}

shouldPersist := false
beaconsList := presenseredis.LoadBeaconsList(client, ctx)
settings := presenseredis.LoadSettings(client, ctx)

for _, beacon := range beaconsList {
length := len(beacon.Beacon_metrics)
if length == 0 {
continue
}

if (int64(time.Now().Unix()) - (beacon.Beacon_metrics[length-1].Timestamp)) > settings.Last_seen_threshold {
if beacon.Expired_location == "expired" {
continue
}
beacon.Expired_location = "expired" // define type expired
} else {
beacon.Expired_location = ""
}

locList := make(map[string]float64)
seenWeight := 1.5
rssiWeight := 0.75

for _, metric := range beacon.Beacon_metrics {
weightCalc := seenWeight + (rssiWeight * (1.0 - (float64(metric.Rssi) / -100.0)))
loc, ok := locList[metric.Location]
if !ok {
loc = weightCalc
} else {
loc = loc + weightCalc
}
}

bestName := ""
ts := 0.0

for name, timesSeen := range locList {
if timesSeen > ts {
bestName = name
ts = timesSeen
}
}

bestLocation := model.BestLocation{Name: bestName, Distance: beacon.Beacon_metrics[length-1].Distance, Last_seen: beacon.Beacon_metrics[length-1].Timestamp}
beacon.Location_history = append(beacon.Location_history, bestName)

if len(beacon.Location_history) > 10 {
beacon.Location_history = beacon.Location_history[1:]
}

locationCounts := make(map[string]int)
for _, loc := range beacon.Location_history {
locationCounts[loc]++
}

maxCount := 0
mostCommonLocation := ""
for loc, count := range locationCounts {
if count > maxCount {
maxCount = count
mostCommonLocation = loc
}
}

if maxCount >= 7 {
beacon.Previous_location = mostCommonLocation
if mostCommonLocation == beacon.Previous_confident_location {
beacon.Location_confidence++
} else {
beacon.Location_confidence = 1
beacon.Previous_confident_location = mostCommonLocation
}
}

r := model.HTTPLocation{
Distance: bestLocation.Distance,
Name: beacon.Name,
Beacon_name: beacon.Name,
Beacon_id: beacon.Beacon_id,
Beacon_type: beacon.Beacon_type,
HB_Battery: beacon.HB_Battery,
HB_ButtonMode: beacon.HB_ButtonMode,
HB_ButtonCounter: beacon.HB_ButtonCounter,
Location: bestName,
Last_seen: bestLocation.Last_seen,
}

if (beacon.Location_confidence == settings.Location_confidence && beacon.Previous_confident_location != bestLocation.Name) || (beacon.Expired_location == "expired") {
shouldPersist = true
beacon.Location_confidence = 0
location := ""
if beacon.Expired_location == "expired" {
location = "expired"
} else {
location = bestName
}
}
}
}

// get likely locations:
/*
1. Locks the http_results list
2. inits list to empty struct type -> TODO: what is this list used for
3. loops through beacons list -> should be locked?
4. check for beacon metrics -> how do you get beacon metrics, I guess it has an array of timestamps
5. check for threshold value in the settings
5.1. check for property expired location
5.2. if location is not expired -> mark it as expired, generate message and send to all clients,
if clients do not respond close the connection
6. Init best location with type Best_location{} -> what is this type
7. make locations list -> key: string, val: float64
7.1 set weight for seen and rssi
7.2 loop over metrics of the beacon -> some alogirthm based on location value

I think the algorithm is recording names of different gateways and their rssi's and then from
that it checks gateway name and makes decisions based on calculated values

7.3 writes result in best location and updates list location history with this name if the list
is longer than 10 elements it removes the first element


*/

+ 0
- 0
cmd/presenSe/.keep 파일 보기


+ 0
- 188
cmd/presenSe/presense.go 파일 보기

@@ -1,188 +0,0 @@
package main

import (
"encoding/json"
"fmt"
"log"
"os"
"os/signal"
"strconv"
"strings"
"time"

"github.com/AFASystems/presence/internal/pkg/config"
"github.com/AFASystems/presence/internal/pkg/httpserver"
"github.com/AFASystems/presence/internal/pkg/model"
"github.com/AFASystems/presence/internal/pkg/mqttclient"
"github.com/AFASystems/presence/internal/pkg/persistence"
"github.com/boltdb/bolt"
"github.com/gorilla/websocket"
"github.com/yosssi/gmq/mqtt"
"github.com/yosssi/gmq/mqtt/client"
)

func main() {
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt)
cfg := config.Load()

fmt.Println("hello world")

db, err := bolt.Open("presence.db", 0644, nil)
if err != nil {
log.Fatal(err)
}
defer db.Close()

model.Db = db

cli := client.New(&client.Options{
ErrorHandler: func(err error) {
fmt.Println(err)
},
})

defer cli.Terminate()

fmt.Println("host: ", cfg.MQTTHost, " Client ID: ", cfg.MQTTClientID, "user: ", cfg.MQTTUser)

err = cli.Connect(&client.ConnectOptions{
Network: "tcp",
Address: cfg.MQTTHost,
ClientID: []byte(cfg.MQTTClientID),
UserName: []byte(cfg.MQTTUser),
Password: []byte(cfg.MQTTPass),
})

if err != nil {
fmt.Println("Error comes from here")
panic(err)
}

ctx := &model.AppContext{
HTTPResults: model.HTTPResultsList{
HTTPResults: model.HTTPLocationsList{Beacons: []model.HTTPLocation{}},
},
Beacons: model.BeaconsList{
Beacons: make(map[string]model.Beacon),
},
ButtonsList: make(map[string]model.Button),
Settings: model.Settings{
Location_confidence: 4,
Last_seen_threshold: 15,
Beacon_metrics_size: 30,
HA_send_interval: 5,
HA_send_changes_only: false,
},
Clients: make(map[*websocket.Conn]bool),
Broadcast: make(chan model.Message, 100),
Locations: model.LocationsList{Locations: make(map[string]model.Location)},
LatestList: model.LatestBeaconsList{LatestList: make(map[string]model.Beacon)},
}

persistence.LoadState(model.Db, ctx)
incomingChan := mqttclient.IncomingMQTTProcessor(1*time.Second, cli, model.Db, ctx)

err = cli.Subscribe(&client.SubscribeOptions{
SubReqs: []*client.SubReq{
&client.SubReq{
TopicFilter: []byte("publish_out/#"),
QoS: mqtt.QoS0,
Handler: func(topicName, message []byte) {
msgStr := string(message)
t := strings.Split(string(topicName), "/")
hostname := t[1]
fmt.Println("hostname: ", hostname)

if strings.HasPrefix(msgStr, "[") {
var readings []model.RawReading
err := json.Unmarshal(message, &readings)
if err != nil {
log.Printf("Error parsing JSON: %v", err)
return
}

for _, reading := range readings {
if reading.Type == "Gateway" {
continue
}
incoming := model.Incoming_json{
Hostname: hostname,
MAC: reading.MAC,
RSSI: int64(reading.RSSI),
Data: reading.RawData,
HB_ButtonCounter: parseButtonState(reading.RawData),
}
incomingChan <- incoming
}
} else {
s := strings.Split(string(message), ",")
if len(s) < 6 {
log.Printf("Messaggio CSV non valido: %s", msgStr)
return
}

rawdata := s[4]
buttonCounter := parseButtonState(rawdata)
if buttonCounter > 0 {
incoming := model.Incoming_json{}
i, _ := strconv.ParseInt(s[3], 10, 64)
incoming.Hostname = hostname
incoming.Beacon_type = "hb_button"
incoming.MAC = s[1]
incoming.RSSI = i
incoming.Data = rawdata
incoming.HB_ButtonCounter = buttonCounter

read_line := strings.TrimRight(string(s[5]), "\r\n")
it, err33 := strconv.Atoi(read_line)
if err33 != nil {
fmt.Println(it)
fmt.Println(err33)
os.Exit(2)
}
incomingChan <- incoming
}
}
},
},
},
})
if err != nil {
panic(err)
}

fmt.Println("CONNECTED TO MQTT")
fmt.Println("\n ")
fmt.Println("Visit http://" + cfg.HTTPAddr + " on your browser to see the web interface")
fmt.Println("\n ")

go httpserver.StartHTTPServer(cfg.HTTPAddr, ctx)

<-sigc

if err := cli.Disconnect(); err != nil {
panic(err)
}
}

func parseButtonState(raw string) int64 {
raw = strings.ToUpper(raw)

if strings.HasPrefix(raw, "0201060303E1FF12") && len(raw) >= 38 {
buttonField := raw[34:38]
if buttonValue, err := strconv.ParseInt(buttonField, 16, 64); err == nil {
return buttonValue
}
}

if strings.HasPrefix(raw, "02010612FF590") && len(raw) >= 24 {
counterField := raw[22:24]
buttonState, err := strconv.ParseInt(counterField, 16, 64)
if err == nil {
return buttonState
}
}

return 0
}

+ 8
- 8
cmd/server/main.go 파일 보기

@@ -32,7 +32,7 @@ func HttpServer(addr string) {
methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS"})

// Kafka writer that relays messages
writer := kafkaclient.KafkaWriter("kafka:9092", "apibeacons")
writer := kafkaclient.KafkaWriter("127.0.0.1:9092", "apibeacons")
defer writer.Close()

settingsWriter := kafkaclient.KafkaWriter("kafka:9092", "settings")
@@ -145,14 +145,14 @@ func beaconsAddHandler(writer *kafka.Writer) http.HandlerFunc {
}
}

func beaconsListHandler(client *redis.Client) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
}
}
// func beaconsListHandler(client *redis.Client) http.HandlerFunc {
// return func(w http.ResponseWriter, r *http.Request) {
// }
// }

func settingsListHandler(client *redis.Client) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {}
}
// func settingsListHandler(client *redis.Client) http.HandlerFunc {
// return func(w http.ResponseWriter, r *http.Request) {}
// }

func settingsEditHandler(writer *kafka.Writer) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {


+ 16
- 0
internal/pkg/model/typeMethods.go 파일 보기

@@ -0,0 +1,16 @@
package model

import (
"crypto/sha256"
"fmt"
)

func (b BeaconEvent) Hash() []byte {
rBatt := (b.Battery / 10) * 10
c := fmt.Sprintf("%d%d%s%s%s", rBatt, b.Event, b.Id, b.Name, b.Type)
h := sha256.New()
h.Write([]byte(c))

bs := h.Sum(nil)
return bs
}

+ 23
- 8
internal/pkg/model/types.go 파일 보기

@@ -127,6 +127,14 @@ type Beacon struct {
HB_ButtonMode string `json:"hb_button_mode"`
}

type BeaconEvent struct {
Name string
Id string
Type string
Battery uint32
Event int
}

// Button represents a hardware button beacon device.
type Button struct {
Name string `json:"name"`
@@ -149,6 +157,11 @@ type BeaconsList struct {
Lock sync.RWMutex
}

type BeaconEventList struct {
Beacons map[string]BeaconEvent
Lock sync.RWMutex
}

// LocationsList holds all known locations with concurrency protection.
type LocationsList struct {
Locations map[string]Location
@@ -186,14 +199,16 @@ type HTTPResultsList struct {
}

type AppContext struct {
HTTPResults HTTPResultsList
Beacons BeaconsList
ButtonsList map[string]Button
Settings Settings
Broadcast chan Message
Locations LocationsList
LatestList LatestBeaconsList
Clients map[*websocket.Conn]bool
HTTPResults HTTPResultsList
Beacons BeaconsList
ButtonsList map[string]Button
Settings Settings
Broadcast chan Message
Locations LocationsList
LatestList LatestBeaconsList
Clients map[*websocket.Conn]bool
BeaconsLookup map[string]struct{}
BeaconEvents BeaconEventList
}

type ApiUpdate struct {


+ 1
- 27
scripts/testAPI.sh 파일 보기

@@ -1,6 +1,6 @@
#!/bin/bash
URL="http://127.0.0.1:1902/api/beacons"
BEACON_ID="E017085443A7"
BEACON_ID="C83F8F17DB35"

echo "POST (create)"
curl -s -X POST $URL \
@@ -10,32 +10,6 @@ echo -e "\n"

sleep 1

echo "GET (list after create)"
curl -s -X GET $URL
echo -e "\n"

sleep 1

echo "PUT (update)"
curl -s -X PUT $URL \
-H "Content-Type: application/json" \
-d '{"Beacon_id":"'"$BEACON_ID"'","Name":"Beacon1-updated","tx_power":-60}'
echo -e "\n"

sleep 1

echo "GET (list after update)"
curl -s -X GET $URL
echo -e "\n"

sleep 1

echo "DELETE"
curl -s -X DELETE "$URL/$BEACON_ID"
echo -e "\n"

sleep 1

echo "GET (list after delete)"
curl -s -X GET $URL
echo -e "\n"

불러오는 중...
취소
저장