Преглед изворни кода

feat: restructure of decoder and bridge

chore/restructure-decoder
Blaz Smehov пре 1 недеља
родитељ
комит
ba93c59a8a
11 измењених фајлова са 70523 додато и 3920 уклоњено
  1. +70299
    -0
      cmd/decoder/analysis.txt
  2. +32
    -0
      cmd/decoder/examples.txt
  3. +144
    -51
      cmd/decoder/main.go
  4. +0
    -3488
      cmd/decoder/save.txt
  5. +0
    -150
      cmd/location/main.go
  6. +0
    -0
      cmd/presenSe/.keep
  7. +0
    -188
      cmd/presenSe/presense.go
  8. +8
    -8
      cmd/server/main.go
  9. +16
    -0
      internal/pkg/model/typeMethods.go
  10. +23
    -8
      internal/pkg/model/types.go
  11. +1
    -27
      scripts/testAPI.sh

+ 70299
- 0
cmd/decoder/analysis.txt
Разлика између датотеке није приказан због своје велике величине
Прегледај датотеку


+ 32
- 0
cmd/decoder/examples.txt Прегледај датотеку

@@ -0,0 +1,32 @@
Ingics iBS01G
id: C83F8F17DB35
020106 12 FF590080BC240100FFFFFFFF000000000000
id: C83F8F17DB35
020106 12 FF590080BC240100FFFFFFFF000000000000


type 0x16 - service Data

Minew B7
id: C300003947C4
020106 0303E1FF 1216E1FFA1031AFFFEFEFB0000C447390000C3
id: C300003947C4
0201061AFF4C000215FDA50693A4E24FB1AFCFC6EB0764782500000000EC - iBeacon
id: C300003947C4
020106 0303AAFE 1516AAFE00E800112233445566778899ABCDE7280002 - eddystone
id: C300003947C4
0201060303E1FF1216E1FFA1031AFFFEFEFB0000C447390000C3
id: C300003947C4
0201060303E1FF0E16E1FFA1081AC447390000C34237

Minew MWB01
id: C7AE561E38B7
02010617FF0001000000000000000000005F0700006F4C0000640003095336
id: C7AE561E38B7
02010617FF00020000FF0000FF0000FF0006001D005200000B200803095336

Minew MWC01
id: E01F9A7A47D2
02010617FF00020000FF0000FF0000FF0006001D005200000A242D03095332
id: E01F9A7A47D2
02010617FF000100000000000000000000780700006F4C0000640003095332

+ 144
- 51
cmd/decoder/main.go Прегледај датотеку

@@ -1,16 +1,22 @@
package main

import (
"bytes"
"context"
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"math"
"strconv"
"strings"
"time"

"github.com/AFASystems/presence/internal/pkg/config"
"github.com/AFASystems/presence/internal/pkg/kafkaclient"
"github.com/AFASystems/presence/internal/pkg/model"
"github.com/AFASystems/presence/internal/pkg/mqttclient"
"github.com/segmentio/kafka-go"
)

func main() {
@@ -19,9 +25,6 @@ func main() {
Beacons: model.BeaconsList{
Beacons: make(map[string]model.Beacon),
},
LatestList: model.LatestBeaconsList{
LatestList: make(map[string]model.Beacon),
},
Settings: model.Settings{
Settings: model.SettingsVal{
Location_confidence: 4,
@@ -31,10 +34,12 @@ func main() {
HA_send_changes_only: false,
},
},
BeaconEvents: model.BeaconEventList{
Beacons: make(map[string]model.BeaconEvent),
},
BeaconsLookup: make(map[string]struct{}),
}

fmt.Println("init")

cfg := config.Load()

// Kafka reader for Raw MQTT beacons
@@ -42,8 +47,13 @@ func main() {
defer rawReader.Close()

// Kafka reader for API server updates
// apiReader := kafkaclient.KafkaReader(cfg.KafkaURL, "apibeacons", "gid-api")
// defer apiReader.Close()
apiReader := kafkaclient.KafkaReader(cfg.KafkaURL, "apibeacons", "gid-api")
defer apiReader.Close()

alertWriter := kafkaclient.KafkaWriter(cfg.KafkaURL, "alertbeacons")
defer alertWriter.Close()

fmt.Println("Decoder initialized, subscribed to Kafka topics")

// // Kafka reader for latest list updates
// latestReader := kafkaclient.KafkaReader(cfg.KafkaURL, "latestbeacons", "gid-latest")
@@ -55,36 +65,27 @@ func main() {

// declare channel for collecting Kafka messages
chRaw := make(chan model.Incoming_json, 2000)
// chApi := make(chan model.ApiUpdate, 2000)
chApi := make(chan model.ApiUpdate, 2000)
// chLatest := make(chan model.Incoming_json, 2000)
// chSettings := make(chan model.SettingsVal, 10)

go kafkaclient.Consume(rawReader, chRaw)
// go kafkaclient.Consume(apiReader, chApi)
go kafkaclient.Consume(apiReader, chApi)
// go kafkaclient.Consume(latestReader, chLatest)
// go kafkaclient.Consume(settingsReader, chSettings)

for {
select {
case msg := <-chRaw:
processIncoming(msg, &appCtx)
// case msg := <-chApi:
// switch msg.Method {
// case "POST":
// fmt.Println("Incoming POST")
// appCtx.Beacons.Lock.Lock()
// appCtx.Beacons.Beacons[msg.Beacon.Beacon_id] = msg.Beacon
// case "DELETE":
// fmt.Println("Incoming delete")
// _, exists := appCtx.Beacons.Beacons[msg.ID]
// if exists {
// appCtx.Beacons.Lock.Lock()
// delete(appCtx.Beacons.Beacons, msg.ID)
// }
// default:
// fmt.Println("unknown method: ", msg.Method)
// }
// appCtx.Beacons.Lock.Unlock()
processIncoming(msg, &appCtx, alertWriter)
case msg := <-chApi:
switch msg.Method {
case "POST":
id := msg.Beacon.Beacon_id
appCtx.BeaconsLookup[id] = struct{}{}
case "DELETE":
fmt.Println("Incoming delete message")
}
// case msg := <-chLatest:
// fmt.Println("latest msg: ", msg)
// case msg := <-chSettings:
@@ -96,44 +97,136 @@ func main() {
}
}

func processIncoming(incoming model.Incoming_json, ctx *model.AppContext) {
defer func() {
if err := recover(); err != nil {
fmt.Println("work failed:", err)
}
}()
func processIncoming(incoming model.Incoming_json, ctx *model.AppContext, writer *kafka.Writer) {
id := mqttclient.GetBeaconID(incoming)
_, ok := ctx.BeaconsLookup[id]
if !ok {
return
}

// Get ID
err := decodeBeacon(incoming, ctx, writer)
if err != nil {
fmt.Println("error in decoding")
return
}
}

func decodeBeacon(incoming model.Incoming_json, ctx *model.AppContext, writer *kafka.Writer) error {
beacon := strings.TrimSpace(incoming.Data)
id := mqttclient.GetBeaconID(incoming)
fmt.Println(incoming.Data)
if beacon == "" {
return nil // How to return error?, do I even need to return error
}

b, err := hex.DecodeString(beacon)
if err != nil {
return err
}

// check for flag byte, if first AD structure is flag bytes, remove it
if len(b) > 1 && b[1] == 0x01 {
l := int(b[0]) // length of AD structure
if 1+l <= len(b) {
b = b[1+l:]
}
}

adStructureIndeces := ParseADFast(b)
event := model.BeaconEvent{}
for _, r := range adStructureIndeces {
ad := b[r[0]:r[1]]
if checkIngics(ad) {
event = parseIngicsState(ad)
event.Id = id
event.Name = id
break
} else if checkEddystoneTLM(ad) {
event = parseEddystoneState(ad)
event.Id = id
event.Name = id
break
} else if checkMinewB7(ad) {
fmt.Println("Minew B7 vendor format")
break
}
}

beacons := &ctx.Beacons
if event.Id != "" {
prevEvent, ok := ctx.BeaconEvents.Beacons[id]
ctx.BeaconEvents.Beacons[id] = event
if ok && bytes.Equal(prevEvent.Hash(), event.Hash()) {
return nil
}

beacons.Lock.Lock()
defer beacons.Lock.Unlock()
eMsg, err := json.Marshal(event)
if err != nil {
return err
}

incoming = mqttclient.IncomingBeaconFilter(incoming)
err = writer.WriteMessages(context.Background(), kafka.Message{
Value: eMsg,
})

beacon, exists := beacons.Beacons[id]
if !exists {
return
if err != nil {
return err
}

fmt.Println("Message sent")
}

fmt.Printf("%+v\n", beacon)
return nil
}

updateBeacon(&beacon, incoming)
beacons.Beacons[id] = beacon
func checkIngics(ad []byte) bool {
if len(ad) >= 6 &&
ad[1] == 0xFF &&
ad[2] == 0x59 &&
ad[3] == 0x00 &&
ad[4] == 0x80 &&
ad[5] == 0xBC {
return true
}

return false
}

func parseIngicsState(ad []byte) model.BeaconEvent {
return model.BeaconEvent{
Battery: uint32(binary.LittleEndian.Uint16(ad[6:8])),
Event: int(ad[8]),
Type: "Ingics",
}
}

func checkEddystoneTLM(ad []byte) bool {
if len(ad) >= 4 &&
ad[1] == 0x16 &&
ad[2] == 0xAA &&
ad[3] == 0xFE &&
ad[4] == 0x20 {
return true
}

return false
}

func processBeacon(hexStr string) {
b, _ := hex.DecodeString(hexStr)
func parseEddystoneState(ad []byte) model.BeaconEvent {
return model.BeaconEvent{
Battery: uint32(binary.BigEndian.Uint16(ad[6:8])),
Type: "Eddystone",
}
}

if len(b) > 2 && b[0] == 0x02 && b[1] == 0x01 {
b = b[2+int(b[0]):]
// I dont think this is always true, but for testing is ok
func checkMinewB7(ad []byte) bool {
if len(ad) >= 4 &&
ad[1] == 0x16 &&
ad[2] == 0xE1 &&
ad[3] == 0xFF {
return true
}

ads := ParseADFast(b)
_ = ads
return false
}

func ParseADFast(b []byte) [][2]int {


+ 0
- 3488
cmd/decoder/save.txt
Разлика између датотеке није приказан због своје велике величине
Прегледај датотеку


+ 0
- 150
cmd/location/main.go Прегледај датотеку

@@ -1,150 +0,0 @@
package main

import (
"context"
"time"

"github.com/AFASystems/presence/internal/pkg/model"
presenseredis "github.com/AFASystems/presence/internal/pkg/redis"
"github.com/redis/go-redis/v9"
)

func main() {
ctx := context.Background()
client := redis.NewClient(&redis.Options{
Addr: "127.0.0.1:6379",
Password: "",
})
getLikelyLocations(client, ctx)
}

func getLikelyLocations(client *redis.Client, ctx context.Context) {
httpRes := model.HTTPResultsList{
HTTPResults: model.HTTPLocationsList{
Beacons: make([]model.HTTPLocation, 0),
},
}

shouldPersist := false
beaconsList := presenseredis.LoadBeaconsList(client, ctx)
settings := presenseredis.LoadSettings(client, ctx)

for _, beacon := range beaconsList {
length := len(beacon.Beacon_metrics)
if length == 0 {
continue
}

if (int64(time.Now().Unix()) - (beacon.Beacon_metrics[length-1].Timestamp)) > settings.Last_seen_threshold {
if beacon.Expired_location == "expired" {
continue
}
beacon.Expired_location = "expired" // define type expired
} else {
beacon.Expired_location = ""
}

locList := make(map[string]float64)
seenWeight := 1.5
rssiWeight := 0.75

for _, metric := range beacon.Beacon_metrics {
weightCalc := seenWeight + (rssiWeight * (1.0 - (float64(metric.Rssi) / -100.0)))
loc, ok := locList[metric.Location]
if !ok {
loc = weightCalc
} else {
loc = loc + weightCalc
}
}

bestName := ""
ts := 0.0

for name, timesSeen := range locList {
if timesSeen > ts {
bestName = name
ts = timesSeen
}
}

bestLocation := model.BestLocation{Name: bestName, Distance: beacon.Beacon_metrics[length-1].Distance, Last_seen: beacon.Beacon_metrics[length-1].Timestamp}
beacon.Location_history = append(beacon.Location_history, bestName)

if len(beacon.Location_history) > 10 {
beacon.Location_history = beacon.Location_history[1:]
}

locationCounts := make(map[string]int)
for _, loc := range beacon.Location_history {
locationCounts[loc]++
}

maxCount := 0
mostCommonLocation := ""
for loc, count := range locationCounts {
if count > maxCount {
maxCount = count
mostCommonLocation = loc
}
}

if maxCount >= 7 {
beacon.Previous_location = mostCommonLocation
if mostCommonLocation == beacon.Previous_confident_location {
beacon.Location_confidence++
} else {
beacon.Location_confidence = 1
beacon.Previous_confident_location = mostCommonLocation
}
}

r := model.HTTPLocation{
Distance: bestLocation.Distance,
Name: beacon.Name,
Beacon_name: beacon.Name,
Beacon_id: beacon.Beacon_id,
Beacon_type: beacon.Beacon_type,
HB_Battery: beacon.HB_Battery,
HB_ButtonMode: beacon.HB_ButtonMode,
HB_ButtonCounter: beacon.HB_ButtonCounter,
Location: bestName,
Last_seen: bestLocation.Last_seen,
}

if (beacon.Location_confidence == settings.Location_confidence && beacon.Previous_confident_location != bestLocation.Name) || (beacon.Expired_location == "expired") {
shouldPersist = true
beacon.Location_confidence = 0
location := ""
if beacon.Expired_location == "expired" {
location = "expired"
} else {
location = bestName
}
}
}
}

// get likely locations:
/*
1. Locks the http_results list
2. inits list to empty struct type -> TODO: what is this list used for
3. loops through beacons list -> should be locked?
4. check for beacon metrics -> how do you get beacon metrics, I guess it has an array of timestamps
5. check for threshold value in the settings
5.1. check for property expired location
5.2. if location is not expired -> mark it as expired, generate message and send to all clients,
if clients do not respond close the connection
6. Init best location with type Best_location{} -> what is this type
7. make locations list -> key: string, val: float64
7.1 set weight for seen and rssi
7.2 loop over metrics of the beacon -> some alogirthm based on location value

I think the algorithm is recording names of different gateways and their rssi's and then from
that it checks gateway name and makes decisions based on calculated values

7.3 writes result in best location and updates list location history with this name if the list
is longer than 10 elements it removes the first element


*/

+ 0
- 0
cmd/presenSe/.keep Прегледај датотеку


+ 0
- 188
cmd/presenSe/presense.go Прегледај датотеку

@@ -1,188 +0,0 @@
package main

import (
"encoding/json"
"fmt"
"log"
"os"
"os/signal"
"strconv"
"strings"
"time"

"github.com/AFASystems/presence/internal/pkg/config"
"github.com/AFASystems/presence/internal/pkg/httpserver"
"github.com/AFASystems/presence/internal/pkg/model"
"github.com/AFASystems/presence/internal/pkg/mqttclient"
"github.com/AFASystems/presence/internal/pkg/persistence"
"github.com/boltdb/bolt"
"github.com/gorilla/websocket"
"github.com/yosssi/gmq/mqtt"
"github.com/yosssi/gmq/mqtt/client"
)

func main() {
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt)
cfg := config.Load()

fmt.Println("hello world")

db, err := bolt.Open("presence.db", 0644, nil)
if err != nil {
log.Fatal(err)
}
defer db.Close()

model.Db = db

cli := client.New(&client.Options{
ErrorHandler: func(err error) {
fmt.Println(err)
},
})

defer cli.Terminate()

fmt.Println("host: ", cfg.MQTTHost, " Client ID: ", cfg.MQTTClientID, "user: ", cfg.MQTTUser)

err = cli.Connect(&client.ConnectOptions{
Network: "tcp",
Address: cfg.MQTTHost,
ClientID: []byte(cfg.MQTTClientID),
UserName: []byte(cfg.MQTTUser),
Password: []byte(cfg.MQTTPass),
})

if err != nil {
fmt.Println("Error comes from here")
panic(err)
}

ctx := &model.AppContext{
HTTPResults: model.HTTPResultsList{
HTTPResults: model.HTTPLocationsList{Beacons: []model.HTTPLocation{}},
},
Beacons: model.BeaconsList{
Beacons: make(map[string]model.Beacon),
},
ButtonsList: make(map[string]model.Button),
Settings: model.Settings{
Location_confidence: 4,
Last_seen_threshold: 15,
Beacon_metrics_size: 30,
HA_send_interval: 5,
HA_send_changes_only: false,
},
Clients: make(map[*websocket.Conn]bool),
Broadcast: make(chan model.Message, 100),
Locations: model.LocationsList{Locations: make(map[string]model.Location)},
LatestList: model.LatestBeaconsList{LatestList: make(map[string]model.Beacon)},
}

persistence.LoadState(model.Db, ctx)
incomingChan := mqttclient.IncomingMQTTProcessor(1*time.Second, cli, model.Db, ctx)

err = cli.Subscribe(&client.SubscribeOptions{
SubReqs: []*client.SubReq{
&client.SubReq{
TopicFilter: []byte("publish_out/#"),
QoS: mqtt.QoS0,
Handler: func(topicName, message []byte) {
msgStr := string(message)
t := strings.Split(string(topicName), "/")
hostname := t[1]
fmt.Println("hostname: ", hostname)

if strings.HasPrefix(msgStr, "[") {
var readings []model.RawReading
err := json.Unmarshal(message, &readings)
if err != nil {
log.Printf("Error parsing JSON: %v", err)
return
}

for _, reading := range readings {
if reading.Type == "Gateway" {
continue
}
incoming := model.Incoming_json{
Hostname: hostname,
MAC: reading.MAC,
RSSI: int64(reading.RSSI),
Data: reading.RawData,
HB_ButtonCounter: parseButtonState(reading.RawData),
}
incomingChan <- incoming
}
} else {
s := strings.Split(string(message), ",")
if len(s) < 6 {
log.Printf("Messaggio CSV non valido: %s", msgStr)
return
}

rawdata := s[4]
buttonCounter := parseButtonState(rawdata)
if buttonCounter > 0 {
incoming := model.Incoming_json{}
i, _ := strconv.ParseInt(s[3], 10, 64)
incoming.Hostname = hostname
incoming.Beacon_type = "hb_button"
incoming.MAC = s[1]
incoming.RSSI = i
incoming.Data = rawdata
incoming.HB_ButtonCounter = buttonCounter

read_line := strings.TrimRight(string(s[5]), "\r\n")
it, err33 := strconv.Atoi(read_line)
if err33 != nil {
fmt.Println(it)
fmt.Println(err33)
os.Exit(2)
}
incomingChan <- incoming
}
}
},
},
},
})
if err != nil {
panic(err)
}

fmt.Println("CONNECTED TO MQTT")
fmt.Println("\n ")
fmt.Println("Visit http://" + cfg.HTTPAddr + " on your browser to see the web interface")
fmt.Println("\n ")

go httpserver.StartHTTPServer(cfg.HTTPAddr, ctx)

<-sigc

if err := cli.Disconnect(); err != nil {
panic(err)
}
}

func parseButtonState(raw string) int64 {
raw = strings.ToUpper(raw)

if strings.HasPrefix(raw, "0201060303E1FF12") && len(raw) >= 38 {
buttonField := raw[34:38]
if buttonValue, err := strconv.ParseInt(buttonField, 16, 64); err == nil {
return buttonValue
}
}

if strings.HasPrefix(raw, "02010612FF590") && len(raw) >= 24 {
counterField := raw[22:24]
buttonState, err := strconv.ParseInt(counterField, 16, 64)
if err == nil {
return buttonState
}
}

return 0
}

+ 8
- 8
cmd/server/main.go Прегледај датотеку

@@ -32,7 +32,7 @@ func HttpServer(addr string) {
methodsOk := handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS"})

// Kafka writer that relays messages
writer := kafkaclient.KafkaWriter("kafka:9092", "apibeacons")
writer := kafkaclient.KafkaWriter("127.0.0.1:9092", "apibeacons")
defer writer.Close()

settingsWriter := kafkaclient.KafkaWriter("kafka:9092", "settings")
@@ -145,14 +145,14 @@ func beaconsAddHandler(writer *kafka.Writer) http.HandlerFunc {
}
}

func beaconsListHandler(client *redis.Client) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
}
}
// func beaconsListHandler(client *redis.Client) http.HandlerFunc {
// return func(w http.ResponseWriter, r *http.Request) {
// }
// }

func settingsListHandler(client *redis.Client) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {}
}
// func settingsListHandler(client *redis.Client) http.HandlerFunc {
// return func(w http.ResponseWriter, r *http.Request) {}
// }

func settingsEditHandler(writer *kafka.Writer) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {


+ 16
- 0
internal/pkg/model/typeMethods.go Прегледај датотеку

@@ -0,0 +1,16 @@
package model

import (
"crypto/sha256"
"fmt"
)

func (b BeaconEvent) Hash() []byte {
rBatt := (b.Battery / 10) * 10
c := fmt.Sprintf("%d%d%s%s%s", rBatt, b.Event, b.Id, b.Name, b.Type)
h := sha256.New()
h.Write([]byte(c))

bs := h.Sum(nil)
return bs
}

+ 23
- 8
internal/pkg/model/types.go Прегледај датотеку

@@ -127,6 +127,14 @@ type Beacon struct {
HB_ButtonMode string `json:"hb_button_mode"`
}

type BeaconEvent struct {
Name string
Id string
Type string
Battery uint32
Event int
}

// Button represents a hardware button beacon device.
type Button struct {
Name string `json:"name"`
@@ -149,6 +157,11 @@ type BeaconsList struct {
Lock sync.RWMutex
}

type BeaconEventList struct {
Beacons map[string]BeaconEvent
Lock sync.RWMutex
}

// LocationsList holds all known locations with concurrency protection.
type LocationsList struct {
Locations map[string]Location
@@ -186,14 +199,16 @@ type HTTPResultsList struct {
}

type AppContext struct {
HTTPResults HTTPResultsList
Beacons BeaconsList
ButtonsList map[string]Button
Settings Settings
Broadcast chan Message
Locations LocationsList
LatestList LatestBeaconsList
Clients map[*websocket.Conn]bool
HTTPResults HTTPResultsList
Beacons BeaconsList
ButtonsList map[string]Button
Settings Settings
Broadcast chan Message
Locations LocationsList
LatestList LatestBeaconsList
Clients map[*websocket.Conn]bool
BeaconsLookup map[string]struct{}
BeaconEvents BeaconEventList
}

type ApiUpdate struct {


+ 1
- 27
scripts/testAPI.sh Прегледај датотеку

@@ -1,6 +1,6 @@
#!/bin/bash
URL="http://127.0.0.1:1902/api/beacons"
BEACON_ID="E017085443A7"
BEACON_ID="C83F8F17DB35"

echo "POST (create)"
curl -s -X POST $URL \
@@ -10,32 +10,6 @@ echo -e "\n"

sleep 1

echo "GET (list after create)"
curl -s -X GET $URL
echo -e "\n"

sleep 1

echo "PUT (update)"
curl -s -X PUT $URL \
-H "Content-Type: application/json" \
-d '{"Beacon_id":"'"$BEACON_ID"'","Name":"Beacon1-updated","tx_power":-60}'
echo -e "\n"

sleep 1

echo "GET (list after update)"
curl -s -X GET $URL
echo -e "\n"

sleep 1

echo "DELETE"
curl -s -X DELETE "$URL/$BEACON_ID"
echo -e "\n"

sleep 1

echo "GET (list after delete)"
curl -s -X GET $URL
echo -e "\n"

Loading…
Откажи
Сачувај