diff --git a/.gitignore b/.gitignore index b89c3d9..f0eb163 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ ui/node_modules ui/build +ui/public/assets/env.js .idea machinery/www yarn.lock machinery/data/config machinery/data/cloud -machinery/data/recordings \ No newline at end of file +machinery/data/recordings +machinery/data/snapshots +machinery/test* \ No newline at end of file diff --git a/machinery/data/snapshots/1661529655.png b/machinery/data/snapshots/1661529655.png deleted file mode 100644 index ac206bd..0000000 Binary files a/machinery/data/snapshots/1661529655.png and /dev/null differ diff --git a/machinery/data/snapshots/1661529661.png b/machinery/data/snapshots/1661529661.png deleted file mode 100644 index 10109a0..0000000 Binary files a/machinery/data/snapshots/1661529661.png and /dev/null differ diff --git a/machinery/data/snapshots/1661529667.png b/machinery/data/snapshots/1661529667.png deleted file mode 100644 index 9ab3eb6..0000000 Binary files a/machinery/data/snapshots/1661529667.png and /dev/null differ diff --git a/machinery/main.go b/machinery/main.go index 9677ae3..e7c2987 100644 --- a/machinery/main.go +++ b/machinery/main.go @@ -10,6 +10,7 @@ import ( "github.com/kerberos-io/agent/machinery/src/log" "github.com/kerberos-io/agent/machinery/src/models" "github.com/kerberos-io/agent/machinery/src/routers" + "github.com/kerberos-io/agent/machinery/src/utils" ) func main() { @@ -56,6 +57,19 @@ func main() { timezone, _ := time.LoadLocation(configuration.Config.Timezone) log.Log.Init(timezone) + // Check if we have a device Key or not, if not + // we will generate one. + if configuration.Config.Key == "" { + key := utils.RandStringBytesMaskImpr(30) + configuration.Config.Key = key + err := components.StoreConfig(configuration.Config) + if err == nil { + log.Log.Info("Main: updated unique key for agent to: " + key) + } else { + log.Log.Info("Main: something went wrong while trying to store key: " + key) + } + } + // Bootstrapping the agent communication := models.Communication{ HandleBootstrap: make(chan string, 1), diff --git a/machinery/src/cloud/Cloud.go b/machinery/src/cloud/Cloud.go index cd8045f..63060a7 100644 --- a/machinery/src/cloud/Cloud.go +++ b/machinery/src/cloud/Cloud.go @@ -2,12 +2,16 @@ package cloud import ( "bytes" + "crypto/tls" "encoding/base64" "encoding/json" "fmt" + "io/ioutil" "sync" + "github.com/gin-gonic/gin" "github.com/kerberos-io/joy4/av/pubsub" + "github.com/minio/minio-go/v6" mqtt "github.com/eclipse/paho.mqtt.golang" av "github.com/kerberos-io/joy4/av" @@ -15,6 +19,7 @@ import ( "gocv.io/x/gocv" "net/http" + "net/url" "runtime" "runtime/debug" "strconv" @@ -288,3 +293,271 @@ func HandleLiveStreamHD(livestreamCursor *pubsub.QueueCursor, configuration *mod } } } + +// VerifyHub godoc +// @Router /api/hub/verify [post] +// @ID verify-hub +// @Security Bearer +// @securityDefinitions.apikey Bearer +// @in header +// @name Authorization +// @Tags config +// @Param config body models.Config true "Config" +// @Summary Will verify the hub connectivity. +// @Description Will verify the hub connectivity. +// @Success 200 {object} models.APIResponse +func VerifyHub(c *gin.Context) { + + var config models.Config + err := c.BindJSON(&config) + + if err == nil { + hubKey := config.HubKey + //hubPrivateKey := config.HubPrivateKey + //hubSite := config.HubSite + hubURI := config.HubURI + + content := []byte(`{"message": "fake-message"}`) + body := bytes.NewReader(content) + req, err := http.NewRequest("POST", hubURI+"/queue/test", body) + if err == nil { + req.Header.Set("X-Kerberos-Cloud-Key", hubKey) + client := &http.Client{} + + resp, err := client.Do(req) + if err == nil { + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err == nil { + if resp.StatusCode == 200 { + c.JSON(200, body) + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while reaching the Kerberos Hub API: " + string(body), + }) + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while ready the response body: " + err.Error(), + }) + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while reaching to the Kerberos Hub API: " + hubURI, + }) + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while creating the HTTP request: " + err.Error(), + }) + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while receiving the config " + err.Error(), + }) + } +} + +// VerifyPersistence godoc +// @Router /api/persistence/verify [post] +// @ID verify-persistence +// @Security Bearer +// @securityDefinitions.apikey Bearer +// @in header +// @name Authorization +// @Tags config +// @Param config body models.Config true "Config" +// @Summary Will verify the persistence. +// @Description Will verify the persistence. +// @Success 200 {object} models.APIResponse +func VerifyPersistence(c *gin.Context) { + + var config models.Config + err := c.BindJSON(&config) + if err != nil || config.Cloud != "" { + + if config.Cloud == "s3" { + + //fmt.Println("Uploading...") + // timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token + // 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4 + // - Timestamp + // - Size + - + microseconds + // - device + // - Region + // - Number of changes + // - Token + + aws_access_key_id := config.S3.Publickey + aws_secret_access_key := config.S3.Secretkey + aws_region := config.S3.Region + + // This is the new way ;) + if config.HubKey != "" { + aws_access_key_id = config.HubKey + } + if config.HubPrivateKey != "" { + aws_secret_access_key = config.HubPrivateKey + } + + s3Client, err := minio.NewWithRegion("s3.amazonaws.com", aws_access_key_id, aws_secret_access_key, true, aws_region) + if err != nil { + c.JSON(400, models.APIResponse{ + Data: "Creation of Kerberos Hub connection failed: " + err.Error(), + }) + } else { + + // Check if we need to use the proxy. + if config.S3.ProxyURI != "" { + var transport http.RoundTripper = &http.Transport{ + Proxy: func(*http.Request) (*url.URL, error) { + return url.Parse(config.S3.ProxyURI) + }, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + s3Client.SetCustomTransport(transport) + } + + deviceKey := "fake-key" + devicename := "justatest" + coordinates := "200-200-400-400" + eventToken := "769" + + timestamp := time.Now().Unix() + fileName := strconv.FormatInt(timestamp, 10) + "_6-967003_justatest_200-200-400-400_24_769.mp4" + content := []byte("test-file") + body := bytes.NewReader(content) + + n, err := s3Client.PutObject(config.S3.Bucket, + config.S3.Username+"/"+fileName, + body, + body.Size(), + minio.PutObjectOptions{ + ContentType: "video/mp4", + StorageClass: "ONEZONE_IA", + UserMetadata: map[string]string{ + "event-timestamp": strconv.FormatInt(timestamp, 10), + "event-microseconds": deviceKey, + "event-instancename": devicename, + "event-regioncoordinates": coordinates, + "event-numberofchanges": deviceKey, + "event-token": eventToken, + "productid": deviceKey, + "publickey": aws_access_key_id, + "uploadtime": "now", + }, + }) + + if err != nil { + c.JSON(400, models.APIResponse{ + Data: "Upload of fake recording failed: " + err.Error(), + }) + } else { + c.JSON(200, models.APIResponse{ + Data: "Upload Finished: file has been uploaded to bucket: " + strconv.FormatInt(n, 10), + }) + } + } + } + + if config.Cloud == "kstorage" { + + uri := config.KStorage.URI + accessKey := config.KStorage.AccessKey + secretAccessKey := config.KStorage.SecretAccessKey + directory := config.KStorage.Directory + provider := config.KStorage.Provider + + if err == nil && uri != "" && accessKey != "" && secretAccessKey != "" { + var postData = []byte(`{"title":"Buy cheese and bread for breakfast."}`) + client := &http.Client{} + req, err := http.NewRequest("POST", uri+"/ping", bytes.NewReader(postData)) + + req.Header.Add("X-Kerberos-Storage-AccessKey", accessKey) + req.Header.Add("X-Kerberos-Storage-SecretAccessKey", secretAccessKey) + resp, err := client.Do(req) + + if err == nil { + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err == nil && resp.StatusCode == http.StatusOK { + + if provider != "" || directory != "" { + + hubKey := config.KStorage.CloudKey + // This is the new way ;) + if config.HubKey != "" { + hubKey = config.HubKey + } + + // Generate a random name. + timestamp := time.Now().Unix() + fileName := strconv.FormatInt(timestamp, 10) + + "_6-967003_justatest_200-200-400-400_24_769.mp4" + content := []byte("test-file") + body := bytes.NewReader(content) + //fileSize := int64(len(content)) + + req, err := http.NewRequest("POST", uri+"/storage", body) + if err == nil { + + req.Header.Set("Content-Type", "video/mp4") + req.Header.Set("X-Kerberos-Storage-CloudKey", hubKey) + req.Header.Set("X-Kerberos-Storage-AccessKey", accessKey) + req.Header.Set("X-Kerberos-Storage-SecretAccessKey", secretAccessKey) + req.Header.Set("X-Kerberos-Storage-Provider", provider) + req.Header.Set("X-Kerberos-Storage-FileName", fileName) + req.Header.Set("X-Kerberos-Storage-Device", "test") + req.Header.Set("X-Kerberos-Storage-Capture", "IPCamera") + req.Header.Set("X-Kerberos-Storage-Directory", directory) + client := &http.Client{} + + resp, err := client.Do(req) + + if err == nil { + if resp != nil { + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err == nil { + if resp.StatusCode == 200 { + c.JSON(200, body) + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while verifying your persistence settings. Make sure your provider is the same as the storage provider in your Kerberos Vault, and the relevant storage provider is configured properly.", + }) + } + } + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Upload of fake recording failed: " + err.Error(), + }) + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while creating /storage POST request." + err.Error(), + }) + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Provider and/or directory is missing from the request.", + }) + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while verifying storage credentials: " + string(body), + }) + } + } else { + c.JSON(400, models.APIResponse{ + Data: "Something went wrong while verifying storage credentials:" + err.Error(), + }) + } + } + } + } else { + c.JSON(400, models.APIResponse{ + Data: "No persistence was specified, so do not know what to verify:" + err.Error(), + }) + } +} diff --git a/machinery/src/components/Config.go b/machinery/src/components/Config.go index 47c3570..04c1410 100644 --- a/machinery/src/components/Config.go +++ b/machinery/src/components/Config.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/base64" "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -157,3 +158,51 @@ func OpenConfig(configuration *models.Configuration) { return } + +func SaveConfig(config models.Config, configuration *models.Configuration, communication *models.Communication) error { + if !communication.IsConfiguring.IsSet() { + communication.IsConfiguring.Set() + + err := StoreConfig(config) + if err != nil { + communication.IsConfiguring.UnSet() + return err + } + + select { + case communication.HandleBootstrap <- "restart": + default: + } + + communication.IsConfiguring.UnSet() + + return nil + } else { + return errors.New("☄ Already reconfiguring") + } +} + +func StoreConfig(config models.Config) error { + // Save into database + if os.Getenv("DEPLOYMENT") == "factory" || os.Getenv("MACHINERY_ENVIRONMENT") == "kubernetes" { + // Write to mongodb + session := database.New().Copy() + defer session.Close() + db := session.DB(database.DatabaseName) + collection := db.C("configuration") + + err := collection.Update(bson.M{ + "type": "config", + "name": os.Getenv("DEPLOYMENT_NAME"), + }, &config) + return err + + // Save into file + } else if os.Getenv("DEPLOYMENT") == "" || os.Getenv("DEPLOYMENT") == "agent" { + res, _ := json.MarshalIndent(config, "", "\t") + err := ioutil.WriteFile("./data/config/config.json", res, 0644) + return err + } + + return errors.New("Not able to update config") +} diff --git a/machinery/src/routers/http/Routes.go b/machinery/src/routers/http/Routes.go index 726bdf1..388c658 100644 --- a/machinery/src/routers/http/Routes.go +++ b/machinery/src/routers/http/Routes.go @@ -1,21 +1,18 @@ package http import ( - "encoding/json" - "io/ioutil" - "os" - jwt "github.com/appleboy/gin-jwt/v2" "github.com/gin-gonic/gin" - "gopkg.in/mgo.v2/bson" + "github.com/kerberos-io/agent/machinery/src/cloud" "github.com/kerberos-io/agent/machinery/src/components" - "github.com/kerberos-io/agent/machinery/src/database" "github.com/kerberos-io/agent/machinery/src/models" ) func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configuration *models.Configuration, communication *models.Communication) *gin.RouterGroup { + // This is legacy should be removed in future! Now everything + // lives under the /api prefix. r.GET("/config", func(c *gin.Context) { c.JSON(200, gin.H{ "config": configuration.Config, @@ -25,37 +22,14 @@ func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configuratio }) }) + // This is legacy should be removed in future! Now everything + // lives under the /api prefix. r.POST("/config", func(c *gin.Context) { - if !communication.IsConfiguring.IsSet() { - communication.IsConfiguring.Set() - - // Save into file - var conf models.Config - err := c.BindJSON(&conf) + var config models.Config + err := c.BindJSON(&config) + if err == nil { + err := components.SaveConfig(config, configuration, communication) if err == nil { - if os.Getenv("DEPLOYMENT") == "factory" || os.Getenv("MACHINERY_ENVIRONMENT") == "kubernetes" { - // Write to mongodb - session := database.New().Copy() - defer session.Close() - db := session.DB(database.DatabaseName) - collection := db.C("configuration") - - collection.Update(bson.M{ - "type": "config", - "name": os.Getenv("DEPLOYMENT_NAME"), - }, &conf) - } else if os.Getenv("DEPLOYMENT") == "" || os.Getenv("DEPLOYMENT") == "agent" { - res, _ := json.MarshalIndent(conf, "", "\t") - ioutil.WriteFile("./data/config/config.json", res, 0644) - } - - select { - case communication.HandleBootstrap <- "restart": - default: - } - - communication.IsConfiguring.UnSet() - c.JSON(200, gin.H{ "data": "☄ Reconfiguring", }) @@ -66,7 +40,7 @@ func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configuratio } } else { c.JSON(400, gin.H{ - "data": "☄ Already reconfiguring", + "data": "Something went wrong: " + err.Error(), }) } }) @@ -84,6 +58,27 @@ func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configuratio }) }) + api.POST("/config", func(c *gin.Context) { + var config models.Config + err := c.BindJSON(&config) + if err == nil { + err := components.SaveConfig(config, configuration, communication) + if err == nil { + c.JSON(200, gin.H{ + "data": "☄ Reconfiguring", + }) + } else { + c.JSON(200, gin.H{ + "data": "☄ Reconfiguring", + }) + } + } else { + c.JSON(400, gin.H{ + "data": "Something went wrong: " + err.Error(), + }) + } + }) + api.GET("/restart", func(c *gin.Context) { communication.HandleBootstrap <- "restart" c.JSON(200, gin.H{ @@ -98,6 +93,14 @@ func AddRoutes(r *gin.Engine, authMiddleware *jwt.GinJWTMiddleware, configuratio }) }) + api.POST("/hub/verify", func(c *gin.Context) { + cloud.VerifyHub(c) + }) + + api.POST("/persistence/verify", func(c *gin.Context) { + cloud.VerifyPersistence(c) + }) + api.Use(authMiddleware.MiddlewareFunc()) { // Secured endpoints.. diff --git a/machinery/src/utils/main.go b/machinery/src/utils/main.go index 5769353..6be052e 100644 --- a/machinery/src/utils/main.go +++ b/machinery/src/utils/main.go @@ -14,6 +14,30 @@ import ( const letterBytes = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = rand.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + + return string(b) +} + func CountDigits(i int64) (count int) { for i != 0 { i /= 10 diff --git a/ui/src/api/agent.js b/ui/src/api/agent.js index 2ddb7ff..dd06853 100644 --- a/ui/src/api/agent.js +++ b/ui/src/api/agent.js @@ -1,7 +1,7 @@ import API from './api'; export function doGetConfig(onSuccess, onError) { - const endpoint = API.get(`config`); + const endpoint = API.get(`api/config`); endpoint .then((res) => { if (res.status !== 200) { @@ -18,7 +18,7 @@ export function doGetConfig(onSuccess, onError) { } export function doSaveConfig(config, onSuccess, onError) { - const endpoint = API.post(`config`, { + const endpoint = API.post(`api/config`, { ...config, }); endpoint @@ -37,7 +37,7 @@ export function doSaveConfig(config, onSuccess, onError) { } export function doGetKerberosAgentTags(onSuccess, onError) { - const endpoint = API.get(`kerberos-agent/tags`); + const endpoint = API.get(`api/kerberos-agent/tags`); endpoint .then((res) => { if (res.status !== 200) { @@ -54,7 +54,7 @@ export function doGetKerberosAgentTags(onSuccess, onError) { } export function doVerifyPersistence(config, onSuccess, onError) { - const endpoint = API.post(`persistence/verify`, { + const endpoint = API.post(`api/persistence/verify`, { ...config, }); endpoint @@ -73,7 +73,7 @@ export function doVerifyPersistence(config, onSuccess, onError) { } export function doVerifyHub(config, onSuccess, onError) { - const endpoint = API.post(`hub/verify`, { + const endpoint = API.post(`api/hub/verify`, { ...config, }); endpoint diff --git a/ui/src/pages/Settings/Settings.jsx b/ui/src/pages/Settings/Settings.jsx index 9bb9053..7f5b795 100644 --- a/ui/src/pages/Settings/Settings.jsx +++ b/ui/src/pages/Settings/Settings.jsx @@ -104,12 +104,6 @@ class Settings extends React.Component { this.onUpdateToggle = this.onUpdateToggle.bind(this); this.onUpdateNumberField = this.onUpdateNumberField.bind(this); this.onUpdateTimeline = this.onUpdateTimeline.bind(this); - this.changeValue = this.changeValue.bind(this); - this.changeVaultValue = this.changeVaultValue.bind(this); - this.changeS3Value = this.changeS3Value.bind(this); - this.changeStorageType = this.changeStorageType.bind(this); - this.changeTimezone = this.changeTimezone.bind(this); - this.filterSettings = this.filterSettings.bind(this); this.verifyPersistenceSettings = this.verifyPersistenceSettings.bind(this); this.verifyHubSettings = this.verifyHubSettings.bind(this); this.calculateTimetable = this.calculateTimetable.bind(this); @@ -253,62 +247,14 @@ class Settings extends React.Component { }); } - changeValue() { - // console.log(this); - } - - changeVaultValue() { - // console.log(this); - } - - changeS3Value() { - // console.log(this); - } - - changeStorageType() { - // console.log(this); - } - - changeTimezone() { - // console.log(this); - } - - filterSettings() { - // console.log(this); - } - - saveGeneralSettings() { - // console.log(this); - } - - saveSTUNTURNSettings() { - // console.log(this); - } - - saveMQTTSettings() { - // console.log(this); - } - - saveHubSettings() { - // console.log(this); - } - - savePersistenceSettings() { - // console.log(this); - } - - verifyPersistenceSettings() { - // console.log(this); - } - - verifyHubSettings() { - // console.log(this); - } - saveConfig() { const { config, dispatchSaveConfig } = this.props; this.setState({ + verifyPersistenceSuccess: false, + verifyPersistenceError: false, + verifyHubSuccess: false, + verifyHubError: false, configSuccess: false, configError: false, }); @@ -332,6 +278,97 @@ class Settings extends React.Component { } } + verifyHubSettings() { + const { config, dispatchVerifyHub } = this.props; + if (config) { + // overriding global for testing. + // hub_key: "xxx" + // hub_private_key: "xxxx" + // hub_site: "testsite" + // hub_uri: "https://api.cloud.kerberos.io" + + this.setState({ + configSuccess: false, + configError: false, + verifyPersistenceSuccess: false, + verifyPersistenceError: false, + verifyHubSuccess: false, + verifyHubError: false, + verifyHubErrorMessage: '', + hubSuccess: false, + hubError: false, + loadingHub: true, + }); + + // .... test fields + + dispatchVerifyHub( + config.config, + () => { + this.setState({ + verifyHubSuccess: true, + verifyHubError: false, + verifyHubErrorMessage: '', + hubSuccess: false, + hubError: false, + loadingHub: false, + }); + }, + (error) => { + this.setState({ + verifyHubSuccess: false, + verifyHubError: true, + verifyHubErrorMessage: error, + hubSuccess: false, + hubError: false, + loadingHub: false, + }); + } + ); + } + } + + verifyPersistenceSettings() { + const { config, dispatchVerifyPersistence } = this.props; + if (config) { + this.setState({ + configSuccess: false, + configError: false, + verifyHubSuccess: false, + verifyHubError: false, + verifyPersistenceSuccess: false, + verifyPersistenceError: false, + persistenceSuccess: false, + persistenceError: false, + loading: true, + }); + + dispatchVerifyPersistence( + config.config, + () => { + this.setState({ + verifyPersistenceSuccess: true, + verifyPersistenceError: false, + verifyPersistenceMessage: '', + persistenceSuccess: false, + persistenceError: false, + loading: false, + }); + }, + (error) => { + this.setState({ + verifyPersistenceSuccess: false, + verifyPersistenceError: true, + verifyPersistenceMessage: error, + persistenceSuccess: false, + persistenceError: false, + loading: false, + }); + } + ); + } + } + render() { const { selectedTab, @@ -1734,8 +1771,8 @@ const mapDispatchToProps = (dispatch /* , ownProps */) => ({ Settings.propTypes = { config: PropTypes.objectOf(PropTypes.object).isRequired, - // dispatchVerifyHub: PropTypes.func.isRequired, - // dispatchVerifyPersistence: PropTypes.func.isRequired, + dispatchVerifyHub: PropTypes.func.isRequired, + dispatchVerifyPersistence: PropTypes.func.isRequired, dispatchGetConfig: PropTypes.func.isRequired, dispatchUpdateConfig: PropTypes.func.isRequired, dispatchSaveConfig: PropTypes.func.isRequired,