refactor mqtt + add configure option

This commit is contained in:
Thomas Quandalle
2022-06-21 22:33:59 +02:00
parent a65e3a45a3
commit faeae1cfe5
12 changed files with 106 additions and 36 deletions
+1
View File
@@ -23,6 +23,7 @@ require (
github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe
github.com/swaggo/gin-swagger v1.5.0
github.com/swaggo/swag v1.8.3
github.com/tevino/abool v1.2.0
gocv.io/x/gocv v0.31.0
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
gopkg.in/natefinch/lumberjack.v2 v2.0.0
+2
View File
@@ -278,6 +278,8 @@ github.com/swaggo/gin-swagger v1.5.0/go.mod h1:3mKpZClKx7mnUGsiwJeEkNhnr1VHMkMaT
github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ=
github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s=
github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg=
github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg=
github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+5 -1
View File
@@ -36,7 +36,9 @@ func PendingUpload() {
}
}
func StartUpload(configuration *models.Configuration, communication *models.Communication) {
func HandleUpload(configuration *models.Configuration, communication *models.Communication) {
log.Log.Debug("HandleUpload: started")
config := configuration.Config
watchDirectory := "./data/cloud/"
@@ -74,6 +76,8 @@ loop:
}
time.Sleep(1 * time.Second)
}
log.Log.Debug("HandleUpload: finished")
}
func HandleHeartBeat(configuration *models.Configuration, communication *models.Communication) {
+9 -9
View File
@@ -18,7 +18,7 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string, d
config.KStorage.Provider == "" ||
config.KStorage.Directory == "" ||
config.KStorage.URI == "" {
log.Log.Info("Kerberos Vault: not properly configured.")
log.Log.Info("UploadKerberosVault: Kerberos Vault not properly configured.")
}
//fmt.Println("Uploading...")
@@ -32,26 +32,26 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string, d
// - Token
// KerberosCloud, this means storage is disabled and proxy enabled.
log.Log.Info("Uploading to Kerberos Vault")
log.Log.Info("UploadKerberosVault: Uploading to Kerberos Vault (" + config.KStorage.URI + ")")
log.Log.Info("Upload started for: " + fileName)
log.Log.Info("UploadKerberosVault: Upload started for " + fileName)
fullname := "data/recordings/" + fileName
file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
defer file.Close()
if err != nil {
log.Log.Info("Upload Failed: file doesn't exists anymore.")
log.Log.Info("UploadKerberosVault: Upload Failed, file doesn't exists anymore.")
os.Remove(directory + "/" + fileName)
return false
}
defer file.Close()
publicKey := config.KStorage.CloudKey
// This is the new way ;)
if config.HubKey != "" {
publicKey = config.HubKey
}
log.Log.Info(config.KStorage.URI)
req, err := http.NewRequest("POST", config.KStorage.URI+"/storage", file)
if err != nil {
log.Log.Error("Error reading request. " + err.Error())
@@ -79,18 +79,18 @@ func UploadKerberosVault(configuration *models.Configuration, fileName string, d
body, err := ioutil.ReadAll(resp.Body)
if err == nil {
if resp.StatusCode == 200 {
log.Log.Info("Upload Finished: " + resp.Status + ", " + string(body))
log.Log.Info("UploadKerberosVault: Upload Finished, " + resp.Status + ", " + string(body))
// We will remove the file from disk as well
os.Remove(fullname)
os.Remove(directory + "/" + fileName)
} else {
log.Log.Info("Upload Failed: " + resp.Status + ", " + string(body))
log.Log.Info("UploadKerberosVault: Upload Failed, " + resp.Status + ", " + string(body))
}
resp.Body.Close()
}
}
} else {
log.Log.Info("Upload Failed: " + err.Error())
log.Log.Info("UploadKerberosVault: Upload Failed, " + err.Error())
}
return true
}
+5 -5
View File
@@ -69,20 +69,20 @@ func UploadS3(configuration *models.Configuration, fileName string, directory st
//numberOfChanges := fileParts[4]
token, _ := strconv.Atoi(fileParts[5])
log.Log.Info("Upload started for: " + fileName)
log.Log.Info("UploadS3: Upload started for " + fileName)
fullname := "data/recordings/" + fileName
file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
defer file.Close()
if err != nil {
log.Log.Error("ERROR: " + err.Error())
log.Log.Error("UploadS3: " + err.Error())
os.Remove(directory + "/" + fileName)
return false
}
fileInfo, err := file.Stat()
if err != nil {
log.Log.Error("ERROR: " + err.Error())
log.Log.Error("UploadS3: " + err.Error())
os.Remove(directory + "/" + fileName)
return false
}
@@ -108,10 +108,10 @@ func UploadS3(configuration *models.Configuration, fileName string, directory st
})
if err != nil {
log.Log.Error("Uploading Failed: " + err.Error())
log.Log.Error("UploadS3: Uploading Failed, " + err.Error())
return false
} else {
log.Log.Info("Upload Finished: file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
log.Log.Info("UploadS3: Upload Finished, file has been uploaded to bucket: " + strconv.FormatInt(n, 10))
os.Remove(directory + "/" + fileName)
return true
}
+15 -4
View File
@@ -11,7 +11,9 @@ import (
"github.com/kerberos-io/agent/machinery/src/computervision"
"github.com/kerberos-io/agent/machinery/src/log"
"github.com/kerberos-io/agent/machinery/src/models"
routers "github.com/kerberos-io/agent/machinery/src/routers/mqtt"
"github.com/kerberos-io/joy4/av/pubsub"
"github.com/tevino/abool"
)
func Bootstrap(configuration *models.Configuration, communication *models.Communication) {
@@ -27,6 +29,7 @@ func Bootstrap(configuration *models.Configuration, communication *models.Commun
communication.HandleUpload = make(chan string, 1)
communication.HandleHeartBeat = make(chan string, 1)
communication.HandleLiveSD = make(chan int64, 1)
communication.IsConfiguring = abool.New()
// Before starting the agent, we have a control goroutine, that might
// do several checks to see if the agent is still operational.
@@ -73,7 +76,7 @@ func RunAgent(configuration *models.Configuration, communication *models.Communi
queue.WriteHeader(streams)
// Configure a MQTT client which helps for a bi-directional communication
mqttClient := ConfigureMQTT(configuration, communication)
mqttClient := routers.ConfigureMQTT(configuration, communication)
// Handle heartbeats
go cloud.HandleHeartBeat(configuration, communication)
@@ -93,7 +96,10 @@ func RunAgent(configuration *models.Configuration, communication *models.Communi
recordingCursor := queue.Oldest()
go capture.HandleRecordStream(recordingCursor, configuration, communication, streams)
//-----------
// Handle Upload to cloud
go cloud.HandleUpload(configuration, communication)
//-------------------------------------------------------------------
// This will go into a blocking state, once this channel is triggered
// the agent will cleanup and restart.
status = <-communication.HandleBootstrap
@@ -101,9 +107,10 @@ func RunAgent(configuration *models.Configuration, communication *models.Communi
// Here we are cleaning up everything!
communication.HandleStream <- "stop"
communication.HandleHeartBeat <- "stop"
communication.HandleUpload <- "stop"
infile.Close()
queue.Close()
DisconnectMQTT(mqttClient)
routers.DisconnectMQTT(mqttClient)
} else {
time.Sleep(time.Second * 2)
@@ -125,7 +132,11 @@ func ControlAgent(communication *models.Communication) {
for {
packetsR := packageCounter.Load().(int64)
if packetsR == previousPacket {
occurence = occurence + 1
// If we are already reconfiguring,
// we dont need to check if the stream is blocking.
if !communication.IsConfiguring.IsSet() {
occurence = occurence + 1
}
} else {
occurence = 0
}
+3 -4
View File
@@ -102,11 +102,11 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi
if config.Capture.Continuous == "true" {
log.Log.Info("Disabled Detecting motion...")
log.Log.Info("ProcessMotion: Continuous recording, so no motion detection.")
} else {
log.Log.Info("Start Detecting motion...")
log.Log.Info("ProcessMotion: Motion detection enabled.")
key := ""
if config.Cloud == "s3" && config.S3.Publickey != "" {
@@ -227,7 +227,7 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi
} else {
detectMotion = false
log.Log.Debug("Disabled: not within time interval.")
log.Log.Debug("ProcessMotion: Time interval not valid, disabling motion detection.")
}
}
@@ -251,7 +251,6 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi
}
runtime.GC()
debug.FreeOSMemory()
log.Log.Info("Stopped motion")
}
log.Log.Debug("ProcessMotion: finished")
+3
View File
@@ -2,6 +2,8 @@ package models
import (
"sync/atomic"
"github.com/tevino/abool"
)
// The communication struct that is managing
@@ -18,4 +20,5 @@ type Communication struct {
HandleLiveHDHandshake chan SDPPayload
HandleLiveHDPeers chan string
HandleONVIF chan OnvifAction
IsConfiguring *abool.AtomicBool
}
+62 -2
View File
@@ -1,18 +1,78 @@
package http
import (
"encoding/json"
"io/ioutil"
"os"
jwt "github.com/appleboy/gin-jwt/v2"
"github.com/gin-gonic/gin"
"gopkg.in/mgo.v2/bson"
"github.com/kerberos-io/agent/machinery/src/components"
"github.com/kerberos-io/agent/machinery/src/database"
"github.com/kerberos-io/agent/machinery/src/models"
)
func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configuration *models.Configuration, communication *models.Communication) *gin.RouterGroup {
r.GET("/config", func(c *gin.Context) {
c.JSON(200, gin.H{
"config": configuration.Config,
"custom": configuration.CustomConfig,
"global": configuration.GlobalConfig,
"snapshot": components.GetSnapshot(),
})
})
r.POST("/config", func(c *gin.Context) {
if !communication.IsConfiguring.IsSet() {
communication.IsConfiguring.Set()
// Save into file
var conf models.Config
var confMap map[string]interface{}
c.BindJSON(&confMap)
inrec, _ := json.Marshal(confMap)
json.Unmarshal(inrec, &conf)
if os.Getenv("DEPLOYMENT") == "" || os.Getenv("DEPLOYMENT") == "agent" {
res, _ := json.MarshalIndent(conf, "", "\t")
ioutil.WriteFile("./data/config/config.json", res, 0644)
} else if os.Getenv("DEPLOYMENT") == "factory" {
// Write to mongodb
session := database.New().Copy()
defer session.Close()
db := session.DB(database.DatabaseName)
collection := db.C("configuration")
collection.Update(bson.M{
"type": "config",
"name": os.Getenv("DEPLOYMENT_NAME"),
}, &conf)
}
configuration.Config = conf // HACK
configuration.CustomConfig = conf // HACK
communication.HandleBootstrap <- "restart"
communication.IsConfiguring.UnSet()
c.JSON(200, gin.H{
"data": "☄ Reconfiguring",
"config": conf,
"custom": conf,
})
} else {
c.JSON(200, gin.H{
"data": "☄ Already reconfiguring",
})
}
})
api := r.Group("/api")
{
api.POST("/login", authMiddleware.LoginHandler)
api.GET("/install", GetInstallation)
api.POST("/install", UpdateInstallation)
api.GET("/config", func(c *gin.Context) {
c.JSON(200, gin.H{
-5
View File
@@ -3,13 +3,8 @@ package routers
import (
"github.com/kerberos-io/agent/machinery/src/models"
"github.com/kerberos-io/agent/machinery/src/routers/http"
"github.com/kerberos-io/agent/machinery/src/routers/mqtt"
)
func StartWebserver(configuration *models.Configuration, communication *models.Communication) {
http.StartServer(configuration, communication)
}
func StartMqttListener(name string) {
mqtt.StartListener(name)
}
-5
View File
@@ -1,5 +0,0 @@
package mqtt
func StartListener(name string) {
}
@@ -1,4 +1,4 @@
package components
package routers
import (
"encoding/json"