From bfb72e8268e5cc21cfa27280381ec36d23c02426 Mon Sep 17 00:00:00 2001 From: Cedric Verstraeten Date: Tue, 17 Jan 2023 19:50:08 +0100 Subject: [PATCH] get rid of gocv --- machinery/data/cloud/.empty | 0 machinery/go.mod | 1 - machinery/go.sum | 4 - machinery/main.go | 6 - machinery/src/capture/USBCamera.go | 55 ----- machinery/src/cloud/Cloud.go | 2 +- machinery/src/computervision/main.go | 258 ++++++++++++------------ machinery/src/routers/websocket/main.go | 2 +- machinery/src/utils/main.go | 12 ++ 9 files changed, 144 insertions(+), 196 deletions(-) delete mode 100644 machinery/data/cloud/.empty delete mode 100644 machinery/src/capture/USBCamera.go diff --git a/machinery/data/cloud/.empty b/machinery/data/cloud/.empty deleted file mode 100644 index e69de29..0000000 diff --git a/machinery/go.mod b/machinery/go.mod index 9918f78..61bacb1 100644 --- a/machinery/go.mod +++ b/machinery/go.mod @@ -28,7 +28,6 @@ require ( github.com/swaggo/gin-swagger v1.5.3 github.com/swaggo/swag v1.8.9 github.com/tevino/abool v1.2.0 - gocv.io/x/gocv v0.32.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) diff --git a/machinery/go.sum b/machinery/go.sum index 09bcc4a..9754b7f 100644 --- a/machinery/go.sum +++ b/machinery/go.sum @@ -109,7 +109,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hybridgroup/mjpeg v0.0.0-20140228234708-4680f319790e/go.mod h1:eagM805MRKrioHYuU7iKLUyFPVKqVV6um5DAvCkUtXs= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -185,7 +184,6 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= @@ -288,8 +286,6 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -gocv.io/x/gocv v0.32.0 h1:KPj/piKjLJQ0oKc0y0GFwTrnNX8zaK++118ereaYQkU= -gocv.io/x/gocv v0.32.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/machinery/main.go b/machinery/main.go index 2516b20..3418697 100644 --- a/machinery/main.go +++ b/machinery/main.go @@ -5,7 +5,6 @@ import ( "os" "time" - "github.com/kerberos-io/agent/machinery/src/capture" "github.com/kerberos-io/agent/machinery/src/components" "github.com/kerberos-io/agent/machinery/src/log" "github.com/kerberos-io/agent/machinery/src/models" @@ -34,11 +33,6 @@ func main() { timeout := os.Args[2] fmt.Println(timeout) - case "usbcamera-test": - - deviceID := os.Args[2] - capture.TestUSBCamera(deviceID) - case "run": { name := os.Args[2] diff --git a/machinery/src/capture/USBCamera.go b/machinery/src/capture/USBCamera.go deleted file mode 100644 index b6c4846..0000000 --- a/machinery/src/capture/USBCamera.go +++ /dev/null @@ -1,55 +0,0 @@ -package capture - -import ( - "strconv" - "time" - - "github.com/kerberos-io/agent/machinery/src/log" - "gocv.io/x/gocv" -) - -func TestUSBCamera(deviceID string) { - webcam, err := gocv.OpenVideoCapture(deviceID) - if err != nil { - log.Log.Error("Error opening video capture device: " + deviceID) - return - } - defer webcam.Close() - buf := gocv.NewMat() - defer buf.Close() - - ok := webcam.Read(&buf) - - if ok { - - now := time.Now().Unix() - saveFile := "./data/capture-test/" + strconv.FormatInt(now, 10) + ".mp4" - fps := 20.0 // webcam.Get(gocv.VideoCaptureFPS) - writer, err := gocv.VideoWriterFile(saveFile, "X264", fps, buf.Cols(), buf.Rows(), true) - if err != nil { - log.Log.Error("error opening video writer device: " + saveFile) - return - } - defer writer.Close() - - //window := gocv.NewWindow("Hello") - log.Log.Info("Start reading device: " + deviceID) - for i := 0; i < 100; i++ { - if ok := webcam.Read(&buf); !ok { - log.Log.Error("Device closed: " + deviceID) - return - } - if buf.Empty() { - continue - } - - writer.Write(buf) - //window.IMShow(buf) - //window.WaitKey(1) - - log.Log.Info("Read frame") - } - - } - log.Log.Info("Done. Close videocapture and recording file.") -} diff --git a/machinery/src/cloud/Cloud.go b/machinery/src/cloud/Cloud.go index 5b401f9..3b9721c 100644 --- a/machinery/src/cloud/Cloud.go +++ b/machinery/src/cloud/Cloud.go @@ -285,7 +285,7 @@ func HandleLiveStreamSD(livestreamCursor *pubsub.QueueCursor, configuration *mod func sendImage(topic string, mqttClient mqtt.Client, pkt av.Packet, decoder *ffmpeg.VideoDecoder, decoderMutex *sync.Mutex) { img, err := capture.DecodeImage(pkt, decoder, decoderMutex) if err == nil { - bytes, _ := computervision.ToBytes(img.Image) + bytes, _ := computervision.ImageToBytes(img.Image) encoded := base64.StdEncoding.EncodeToString(bytes) mqttClient.Publish(topic, 0, false, encoded) } diff --git a/machinery/src/computervision/main.go b/machinery/src/computervision/main.go index 00975bb..69ed8c0 100644 --- a/machinery/src/computervision/main.go +++ b/machinery/src/computervision/main.go @@ -3,13 +3,12 @@ package computervision import ( "bytes" "image" + "image/draw" "image/jpeg" "io/ioutil" + "math" "os" - "runtime" - "runtime/debug" "sort" - "strconv" "sync" "time" @@ -22,92 +21,40 @@ import ( geo "github.com/kellydunn/golang-geo" "github.com/kerberos-io/joy4/av" "github.com/kerberos-io/joy4/cgo/ffmpeg" - "gocv.io/x/gocv" ) -func GetRGBImage(pkt av.Packet, dec *ffmpeg.VideoDecoder, decoderMutex *sync.Mutex) gocv.Mat { - var rgb gocv.Mat - img, err := capture.DecodeImage(pkt, dec, decoderMutex) - if err == nil && img != nil { - rgb, _ = ToRGB8(img.Image) - gocv.Resize(rgb, &rgb, image.Pt(rgb.Cols()/4, rgb.Rows()/4), 0, 0, gocv.InterpolationArea) - } - return rgb +func GetImageImage(pkt av.Packet, dec *ffmpeg.VideoDecoder, decoderMutex *sync.Mutex) (image.YCbCr, error) { + rgb, err := capture.DecodeImage(pkt, dec, decoderMutex) + return rgb.Image, err } -func GetImage(pkt av.Packet, dec *ffmpeg.VideoDecoder, decoderMutex *sync.Mutex) gocv.Mat { - var rgb gocv.Mat - img, err := capture.DecodeImage(pkt, dec, decoderMutex) - - if err == nil && img != nil { - - // Check if we need to scale down. - width := img.Width() - height := img.Height() - newWidth := width - newHeight := height - - // Try minify twice. - scaleFactor := 1.0 - if newWidth > 800 { - newWidth = width / 2 - newHeight = height / 2 - scaleFactor *= 2 - } - if newWidth > 800 { - newWidth = width / 2 - newHeight = height / 2 - scaleFactor *= 2 - } - if newWidth > 800 { - newWidth = width / 2 - newHeight = height / 2 - scaleFactor *= 2 - } - - im := img.Image - bounds := im.Bounds() - x := bounds.Dx() - y := bounds.Dy() - if x > 0 && y > 0 { - rgb, _ = ToRGB8(im) - img.Free() - if scaleFactor > 1 { - gocv.Resize(rgb, &rgb, image.Pt(newWidth, newHeight), 0, 0, gocv.InterpolationArea) - } +func ImageThreshold(img *image.Gray, threshold uint8) *image.Gray { + for i := 0; i < len(img.Pix); i++ { + if img.Pix[i] < threshold { + img.Pix[i] = 0 + } else { + img.Pix[i] = 255 } } - return rgb + return img } -func ToGray(rgb gocv.Mat) (gocv.Mat, error) { - gray := gocv.NewMat() - gocv.CvtColor(rgb, &gray, gocv.ColorBGRToGray) - rgb.Close() - return gray, nil +func ImageToGray(img image.YCbCr) *image.Gray { + result := image.NewGray(img.Bounds()) + draw.Draw(result, result.Bounds(), &img, img.Bounds().Min, draw.Src) + return result } -func ToBytes(img image.YCbCr) ([]byte, error) { +func ToBytes(img *image.Gray) ([]byte, error) { buffer := new(bytes.Buffer) - err := jpeg.Encode(buffer, &img, &jpeg.Options{Quality: 100}) + err := jpeg.Encode(buffer, img, &jpeg.Options{Quality: 100}) return buffer.Bytes(), err } -func ToRGB8(img image.YCbCr) (gocv.Mat, error) { - bounds := img.Bounds() - x := bounds.Dx() - y := bounds.Dy() - bytes := make([]byte, 0, x*y*3) - for j := bounds.Min.Y; j < bounds.Max.Y; j++ { - for i := bounds.Min.X; i < bounds.Max.X; i++ { - iy := img.At(i, j) - if iy != nil { - r, g, b, _ := iy.RGBA() - bytes = append(bytes, byte(b>>8), byte(g>>8), byte(r>>8)) - } - } - } - return gocv.NewMatFromBytes(y, x, gocv.MatTypeCV8UC3, bytes) +func ImageToBytes(img image.YCbCr) ([]byte, error) { + buffer := new(bytes.Buffer) + err := jpeg.Encode(buffer, &img, &jpeg.Options{Quality: 100}) + return buffer.Bytes(), err } func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, decoder *ffmpeg.VideoDecoder, decoderMutex *sync.Mutex) { //, wg *sync.WaitGroup) { @@ -128,7 +75,9 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi key := config.HubKey // Initialise first 2 elements - var matArray [3]*gocv.Mat + //var matArray [3]*gocv.Mat + var imageArray [3]*image.Gray + j := 0 //for pkt := range packets { @@ -139,17 +88,22 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi pkt, cursorError = motionCursor.ReadPacket() // Check If valid package. if len(pkt.Data) > 0 && pkt.IsKeyFrame { - rgb := GetImage(pkt, decoder, decoderMutex) - gray, _ := ToGray(rgb) - matArray[j] = &gray - j++ + rgbImage, err := GetImageImage(pkt, decoder, decoderMutex) + if err == nil { + grayImage := ImageToGray(rgbImage) + imageArray[j] = grayImage + //rgb := GetImage(pkt, decoder, decoderMutex) + //gray, _ := ToGray(rgb) + //matArray[j] = &gray + j++ + } } if j == 2 { break } } - img := matArray[0] + img := imageArray[0] if img != nil { // Calculate mask @@ -168,8 +122,9 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi polyObjects = append(polyObjects, poly) } - rows := img.Rows() - cols := img.Cols() + bounds := img.Bounds() + rows := bounds.Dx() + cols := bounds.Dy() var coordinatesToCheck [][]int for y := 0; y < rows; y++ { for x := 0; x < cols; x++ { @@ -195,9 +150,14 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi continue } - rgb := GetImage(pkt, decoder, decoderMutex) - gray, _ := ToGray(rgb) - matArray[2] = &gray + //rgb := GetImage(pkt, decoder, decoderMutex) + //gray, _ := ToGray(rgb) + //matArray[2] = &gray + rgbImage, err := GetImageImage(pkt, decoder, decoderMutex) + if err == nil { + grayImage := ImageToGray(rgbImage) + imageArray[2] = grayImage + } // Store snapshots (jpg) or hull. files, err := ioutil.ReadDir("./data/snapshots") @@ -209,10 +169,14 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi os.Remove("./data/snapshots/" + files[0].Name()) } } - t := strconv.FormatInt(time.Now().Unix(), 10) - snapshotRGB := GetImage(pkt, decoder, decoderMutex) - gocv.IMWrite("./data/snapshots/"+t+".png", snapshotRGB) - snapshotRGB.Close() + + // Todo snapshot + //rgbImage, err := GetImageImage(pkt, decoder, decoderMutex) + + //t := strconv.FormatInt(time.Now().Unix(), 10) + //snapshotRGB := GetImage(pkt, decoder, decoderMutex) + //gocv.IMWrite("./data/snapshots/"+t+".png", snapshotRGB) + //snapshotRGB.Close() // Check if continuous recording. if config.Capture.Continuous == "true" { @@ -246,7 +210,7 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi } // Remember additional information about the result of findmotion - isPixelChangeThresholdReached, changesToReturn = FindMotion(matArray, coordinatesToCheck, config.Capture.PixelChangeThreshold) + isPixelChangeThresholdReached, changesToReturn = FindMotion(imageArray, coordinatesToCheck, config.Capture.PixelChangeThreshold) if detectMotion && isPixelChangeThresholdReached { @@ -263,61 +227,99 @@ func ProcessMotion(motionCursor *pubsub.QueueCursor, configuration *models.Confi } } - matArray[0].Close() - matArray[0] = matArray[1] - matArray[1] = matArray[2] + imageArray[0] = imageArray[1] + imageArray[1] = imageArray[2] i++ - runtime.GC() - debug.FreeOSMemory() + } + + if img != nil { + img = nil } } - if img != nil { - img.Close() - } - runtime.GC() - debug.FreeOSMemory() } log.Log.Debug("ProcessMotion: finished") } -func FindMotion(matArray [3]*gocv.Mat, coordinatesToCheck [][]int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int) { +func FindMotion(imageArray [3]*image.Gray, coordinatesToCheck [][]int, pixelChangeThreshold int) (thresholdReached bool, changesDetected int) { - h1 := gocv.NewMat() - gocv.AbsDiff(*matArray[2], *matArray[0], &h1) - h2 := gocv.NewMat() - gocv.AbsDiff(*matArray[2], *matArray[1], &h2) + image1 := imageArray[0] + image2 := imageArray[1] + image3 := imageArray[2] - and := gocv.NewMat() - gocv.BitwiseAnd(h1, h2, &and) - h1.Close() - h2.Close() + // Calculate the absolute difference between the first and second image. + diff1 := AbsDiff(image3, image1) + diff2 := AbsDiff(image3, image2) - thresh := gocv.NewMat() - gocv.Threshold(and, &thresh, 30.0, 255.0, gocv.ThresholdBinary) - and.Close() + // Calculate the bitwise AND between the first and second image. + and := BitwiseAnd(diff1, diff2) - kernel := gocv.GetStructuringElement(gocv.MorphRect, image.Pt(3, 3)) - eroded := gocv.NewMat() - gocv.Erode(thresh, &eroded, kernel) - thresh.Close() - kernel.Close() + // Do threshold§ + threshold := Threshold(and, 30) + + // Erode and dilate the image to remove noise. + erode := Erode(threshold, 3) changes := 0 for _, c := range coordinatesToCheck { - value := eroded.GetUCharAt(c[1], c[0]) + pixel := c[0] + c[1]*erode.Stride + value := erode.Pix[pixel] if value > 0 { changes++ } } - eroded.Close() - log.Log.Info("FindMotion: Number of changes detected:" + strconv.Itoa(changes)) - - if pixelChangeThreshold == 0 { - pixelChangeThreshold = 75 // Keep hardcoded value of 75 for now if no value is given for changes treshold in config.json - } - - changesDetected = changes // Assign final amount of changes to the return variable - return changes > pixelChangeThreshold, changesDetected // Return bool ifReachedThreshold AND the amount of changes detected in total + return changes > pixelChangeThreshold, changes +} + +func Threshold(img image.Gray, threshold int) (thresholded image.Gray) { + thresholded = image.Gray{ + Pix: make([]uint8, len(img.Pix)), + Stride: img.Stride, + Rect: img.Rect, + } + for i := 0; i < len(img.Pix); i++ { + if int(img.Pix[i]) > threshold { + thresholded.Pix[i] = 255 + } else { + thresholded.Pix[i] = 0 + } + } + return thresholded +} + +func AbsDiff(img1 *image.Gray, img2 *image.Gray) (diff image.Gray) { + diff = image.Gray{ + Pix: make([]uint8, len(img1.Pix)), + Stride: img1.Stride, + Rect: img1.Rect, + } + for i := 0; i < len(img1.Pix); i++ { + diff.Pix[i] = uint8(math.Abs(float64(img1.Pix[i]) - float64(img2.Pix[i]))) + } + return diff +} + +func BitwiseAnd(img1 image.Gray, img2 image.Gray) (and image.Gray) { + and = image.Gray{ + Pix: make([]uint8, len(img1.Pix)), + Stride: img1.Stride, + Rect: img1.Rect, + } + for i := 0; i < len(img1.Pix); i++ { + and.Pix[i] = img1.Pix[i] & img2.Pix[i] + } + return and +} + +func Erode(img image.Gray, kernelSize int) (eroded image.Gray) { + eroded = image.Gray{ + Pix: make([]uint8, len(img.Pix)), + Stride: img.Stride, + Rect: img.Rect, + } + for i := 0; i < len(img.Pix); i++ { + eroded.Pix[i] = img.Pix[i] + } + return eroded } diff --git a/machinery/src/routers/websocket/main.go b/machinery/src/routers/websocket/main.go index ff40104..fe0c5c2 100644 --- a/machinery/src/routers/websocket/main.go +++ b/machinery/src/routers/websocket/main.go @@ -141,7 +141,7 @@ logreader: } img, err := capture.DecodeImage(pkt, decoder, decoderMutex) if err == nil { - bytes, _ := computervision.ToBytes(img.Image) + bytes, _ := computervision.ImageToBytes(img.Image) encodedImage = base64.StdEncoding.EncodeToString(bytes) } } else { diff --git a/machinery/src/utils/main.go b/machinery/src/utils/main.go index 20fce57..a12ef84 100644 --- a/machinery/src/utils/main.go +++ b/machinery/src/utils/main.go @@ -20,6 +20,18 @@ import ( const letterBytes = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +// MaxUint8 - maximum value which can be held in an uint8 +const MaxUint8 = ^uint8(0) + +// MinUint8 - minimum value which can be held in an uint8 +const MinUint8 = 0 + +// MaxUint16 - maximum value which can be held in an uint16 +const MaxUint16 = ^uint16(0) + +// MinUint16 - minimum value which can be held in an uint8 +const MinUint16 = 0 + const ( letterIdxBits = 6 // 6 bits to represent a letter index letterIdxMask = 1<