fix: enabling backchannel on the mainstream

machinery/go.mod

@@ -2,7 +2,7 @@ module github.com/kerberos-io/agent/machinery
 
 go 1.19
 
-//replace github.com/kerberos-io/joy4 v1.0.60 => ../../../../github.com/kerberos-io/joy4
+//replace github.com/kerberos-io/joy4 v1.0.63 => ../../../../github.com/kerberos-io/joy4
 
 // replace github.com/kerberos-io/onvif v0.0.6 => ../../../../github.com/kerberos-io/onvif
 
@@ -26,7 +26,7 @@ require (
 	github.com/golang-module/carbon/v2 v2.2.3
 	github.com/gorilla/websocket v1.5.0
 	github.com/kellydunn/golang-geo v0.7.0
-	github.com/kerberos-io/joy4 v1.0.62
+	github.com/kerberos-io/joy4 v1.0.64
 	github.com/kerberos-io/onvif v0.0.7
 	github.com/minio/minio-go/v6 v6.0.57
 	github.com/nsmith5/mjpeg v0.0.0-20200913181537-54b8ada0e53e

machinery/go.sum

@@ -264,8 +264,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kellydunn/golang-geo v0.7.0 h1:A5j0/BvNgGwY6Yb6inXQxzYwlPHc6WVZR+MrarZYNNg=
 github.com/kellydunn/golang-geo v0.7.0/go.mod h1:YYlQPJ+DPEzrHx8kT3oPHC/NjyvCCXE+IuKGKdrjrcU=
-github.com/kerberos-io/joy4 v1.0.62 h1:LsjGrss5I2UGfTovAF0icTuEcxwOPptkSqGyxeIwa40=
-github.com/kerberos-io/joy4 v1.0.62/go.mod h1:nZp4AjvKvTOXRrmDyAIOw+Da+JA5OcSo/JundGfOlFU=
+github.com/kerberos-io/joy4 v1.0.64 h1:gTUSotHSOhp9mNqEecgq88tQHvpj7TjmrvPUsPm0idg=
+github.com/kerberos-io/joy4 v1.0.64/go.mod h1:nZp4AjvKvTOXRrmDyAIOw+Da+JA5OcSo/JundGfOlFU=
 github.com/kerberos-io/onvif v0.0.7 h1:LIrXjTH7G2W9DN69xZeJSB0uS3W1+C3huFO8kTqx7/A=
 github.com/kerberos-io/onvif v0.0.7/go.mod h1:Hr2dJOH2LM5SpYKk17gYZ1CMjhGhUl+QlT5kwYogrW0=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=

machinery/src/capture/IPCamera.go

@@ -16,12 +16,27 @@ import (
 	"github.com/kerberos-io/joy4/format"
 )
 
-func OpenRTSP(ctx context.Context, url string) (av.DemuxCloser, []av.CodecData, error) {
+func OpenRTSP(ctx context.Context, url string, withBackChannel bool) (av.DemuxCloser, []av.CodecData, error) {
 	format.RegisterAll()
-	infile, err := avutil.Open(ctx, url)
+
+	// Try with backchannel first (if variable is set to true)
+	// If set to true, it will try to open the stream with a backchannel
+	// If fails we will try again (see below).
+	infile, err := avutil.Open(ctx, url, withBackChannel)
 	if err == nil {
 		streams, errstreams := infile.Streams()
-		return infile, streams, errstreams
+		if len(streams) > 0 {
+			return infile, streams, errstreams
+		} else {
+			// Try again without backchannel
+			log.Log.Info("OpenRTSP: trying without backchannel")
+			withBackChannel = false
+			infile, err := avutil.Open(ctx, url, withBackChannel)
+			if err == nil {
+				streams, errstreams := infile.Streams()
+				return infile, streams, errstreams
+			}
+		}
 	}
 	return nil, []av.CodecData{}, err
 }

machinery/src/capture/main.go

@@ -469,7 +469,7 @@ func VerifyCamera(c *gin.Context) {
 		if streamType == "secondary" {
 			rtspUrl = cameraStreams.SubRTSP
 		}
-		_, codecs, err := OpenRTSP(ctx, rtspUrl)
+		_, codecs, err := OpenRTSP(ctx, rtspUrl, true)
 		if err == nil {
 
 			videoIdx := -1

machinery/src/components/Kerberos.go

@@ -118,9 +118,10 @@ func RunAgent(configDirectory string, configuration *models.Configuration, commu
 	status := "not started"
 
 	// Currently only support H264 encoded cameras, this will change.
-	// Establishing the camera connection
+	// Establishing the camera connection without backchannel if no substream
 	rtspUrl := config.Capture.IPCamera.RTSP
-	infile, streams, err := capture.OpenRTSP(context.Background(), rtspUrl)
+	withBackChannel := true
+	infile, streams, err := capture.OpenRTSP(context.Background(), rtspUrl, withBackChannel)
 
 	// We will initialise the camera settings object
 	// so we can check if the camera settings have changed, and we need
@@ -157,7 +158,8 @@ func RunAgent(configDirectory string, configuration *models.Configuration, commu
 		subStreamEnabled := false
 		subRtspUrl := config.Capture.IPCamera.SubRTSP
 		if subRtspUrl != "" && subRtspUrl != rtspUrl {
-			subInfile, subStreams, err = capture.OpenRTSP(context.Background(), subRtspUrl)
+			withBackChannel := false
+			subInfile, subStreams, err = capture.OpenRTSP(context.Background(), subRtspUrl, withBackChannel) // We'll try to enable backchannel for the substream.
 			if err == nil {
 				log.Log.Info("RunAgent: opened RTSP sub stream " + subRtspUrl)
 				subStreamEnabled = true

machinery/src/onvif/main.go

@@ -751,24 +751,6 @@ func GoToPresetFromDevice(device *onvif.Device, presetName string) error {
 	return err
 }
 
-func getXMLNode(xmlBody string, nodeName string) (*xml.Decoder, *xml.StartElement, error) {
-	xmlBytes := bytes.NewBufferString(xmlBody)
-	decodedXML := xml.NewDecoder(xmlBytes)
-	for {
-		token, err := decodedXML.Token()
-		if err != nil {
-			break
-		}
-		switch et := token.(type) {
-		case xml.StartElement:
-			if et.Name.Local == nodeName {
-				return decodedXML, &et, nil
-			}
-		}
-	}
-	return nil, nil, errors.New("error in NodeName - username and password might be wrong")
-}
-
 func GetPTZFunctionsFromDevice(configurations ptz.GetConfigurationsResponse) ([]string, bool, bool) {
 	var functions []string
 	canZoom := false
@@ -854,3 +836,21 @@ func VerifyOnvifConnection(c *gin.Context) {
 		})
 	}
 }
+
+func getXMLNode(xmlBody string, nodeName string) (*xml.Decoder, *xml.StartElement, error) {
+	xmlBytes := bytes.NewBufferString(xmlBody)
+	decodedXML := xml.NewDecoder(xmlBytes)
+	for {
+		token, err := decodedXML.Token()
+		if err != nil {
+			break
+		}
+		switch et := token.(type) {
+		case xml.StartElement:
+			if et.Name.Local == nodeName {
+				return decodedXML, &et, nil
+			}
+		}
+	}
+	return nil, nil, errors.New("error in NodeName - username and password might be wrong")
+}

machinery/src/routers/mqtt/main.go

@@ -476,14 +476,9 @@ func HandleReceiveHDCandidates(mqttClient mqtt.Client, hubKey string, payload mo
 
 	if receiveHDCandidatesPayload.Timestamp != 0 {
 		if communication.CameraConnected {
+			// Register candidate channel
 			key := configuration.Config.Key + "/" + receiveHDCandidatesPayload.SessionID
-			channel := webrtc.CandidateArrays[key]
-			if channel == nil {
-				channel = make(chan string)
-				webrtc.CandidateArrays[key] = channel
-			}
-			log.Log.Info("HandleReceiveHDCandidates: " + receiveHDCandidatesPayload.Candidate)
-			channel <- receiveHDCandidatesPayload.Candidate
+			webrtc.RegisterCandidates(key, receiveHDCandidatesPayload)
 		} else {
 			log.Log.Info("HandleReceiveHDCandidates: received candidate, but camera is not connected.")
 		}

machinery/src/webrtc/main.go

@@ -87,6 +87,21 @@ func (w WebRTC) CreateOffer(sd []byte) pionWebRTC.SessionDescription {
 	return offer
 }
 
+func RegisterCandidates(key string, candidate models.ReceiveHDCandidatesPayload) {
+
+	// Set lock
+	CandidatesMutex.Lock()
+	defer CandidatesMutex.Unlock()
+
+	channel := CandidateArrays[key]
+	if channel == nil {
+		channel = make(chan string)
+		CandidateArrays[key] = channel
+	}
+	log.Log.Info("HandleReceiveHDCandidates: " + candidate.Candidate)
+	channel <- candidate.Candidate
+}
+
 func InitializeWebRTCConnection(configuration *models.Configuration, communication *models.Communication, mqttClient mqtt.Client, videoTrack *pionWebRTC.TrackLocalStaticSample, audioTrack *pionWebRTC.TrackLocalStaticSample, handshake models.RequestHDStreamPayload, candidates chan string) {
 
 	config := configuration.Config
@@ -145,6 +160,9 @@ func InitializeWebRTCConnection(configuration *models.Configuration, communicati
 
 			peerConnection.OnICEConnectionStateChange(func(connectionState pionWebRTC.ICEConnectionState) {
 				if connectionState == pionWebRTC.ICEConnectionStateDisconnected {
+					CandidatesMutex.Lock()
+					defer CandidatesMutex.Unlock()
+
 					atomic.AddInt64(&peerConnectionCount, -1)
 					peerConnections[handshake.SessionID] = nil
 					close(candidates)