change to different folder structure to support kustomize and overlays to define deployment layer

.github/workflows/kind-cloud.yaml

@@ -37,7 +37,7 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           kubectl create namespace mongodb
-          helm install mongodb -n mongodb bitnami/mongodb --values ./mongodb-values.yaml
+          helm install mongodb -n mongodb bitnami/mongodb --values ./base/mongodb-values.yaml
           echo "Sleeping for 250 seconds, give time for the helm chart to create the pods" && sleep 250
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep mongodb | awk '{print $3}' | grep -q '1/1' && echo "mongodb pod is running with status 1/1" || (echo "mongodb pod is not running with status 1/1" && exit 1)
@@ -53,9 +53,9 @@ jobs:
         id: install-kerberos-vault
         run: |
           kubectl create namespace kerberos-vault
-          kubectl apply -f ./mongodb-configmap.yaml -n kerberos-vault
-          kubectl apply -f ./kerberos-vault-deployment.yaml -n kerberos-vault
-          kubectl apply -f ./kerberos-vault-service.yaml -n kerberos-vault
+          kubectl apply -f ./base/mongodb-configmap.yaml -n kerberos-vault
+          kubectl apply -f ./base/vault/kerberos-vault-deployment.yaml -n kerberos-vault
+          kubectl apply -f ./base/vault/kerberos-vault-service.yaml -n kerberos-vault
           echo "Sleeping for 30 seconds, give time for the helm chart to create the pods" && sleep 30
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep kerberos-vault | awk '{print $3}' | grep -q '1/1' && echo "kerberos-vault pod is running with status 1/1" || (echo "kerberos-vault pod is not running with status 1/1" && exit 1)
@@ -80,7 +80,7 @@ jobs:
         run: |
           kubectl create namespace vernemq
           helm repo add vernemq https://vernemq.github.io/docker-vernemq
-          helm install vernemq vernemq/vernemq --values ./vernemq-values.yaml -n vernemq --create-namespace
+          helm install vernemq vernemq/vernemq --values ./base/vernemq-values.yaml -n vernemq --create-namespace
           echo "Sleeping for 120 seconds, give time for the helm chart to create the pods" && sleep 120
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep vernemq-0 | awk '{print $3}' | grep -q '1/1' && echo "vernemq pod is running with status 1/1" || (echo "vernemq pod is not running with status 1/1" && exit 1)
@@ -90,7 +90,7 @@ jobs:
         run: |
           helm repo add kerberos https://charts.kerberos.io
           kubectl create namespace kerberos-hub
-          helm install hub kerberos/hub --values kerberos-hub-values.yaml -n kerberos-hub --create-namespace
+          helm install hub kerberos/hub --values base/hub/kerberos-hub-values.yaml -n kerberos-hub --create-namespace
           echo "Sleeping for 300 seconds, give time for the helm chart to create the pods" && sleep 300
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep kerberos-hub | awk '{print $3}' | grep -q '1/1' && echo "kerberos-hub pod is running with status 1/1" || (echo "kerberos-hub pod is not running with status 1/1" && exit 1)

.github/workflows/kind.yaml

@@ -45,7 +45,7 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           kubectl create namespace mongodb
-          helm install mongodb -n mongodb bitnami/mongodb --values ./mongodb-values.yaml
+          helm install mongodb -n mongodb bitnami/mongodb --values ./base/mongodb-values.yaml
           echo "Sleeping for 250 seconds, give time for the helm chart to create the pods" && sleep 250
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep mongodb | awk '{print $3}' | grep -q '1/1' && echo "mongodb pod is running with status 1/1" || (echo "mongodb pod is not running with status 1/1" && exit 1)
@@ -54,7 +54,7 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           kubectl create namespace rabbitmq
-          helm install rabbitmq -n rabbitmq bitnami/rabbitmq --values ./rabbitmq-values.yaml
+          helm install rabbitmq -n rabbitmq bitnami/rabbitmq --values ./base/rabbitmq-values.yaml
           echo "Sleeping for 60 seconds, give time for the helm chart to create the pods" && sleep 60
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep rabbitmq | awk '{print $3}' | grep -q '1/1' && echo "rabbitmq pod is running with status 1/1" || (echo "rabbitmq pod is not running with status 1/1" && exit 1)
@@ -62,26 +62,26 @@ jobs:
         id: install-kerberos-vault
         run: |
           kubectl create namespace kerberos-vault
-          kubectl apply -f ./mongodb-configmap.yaml -n kerberos-vault
-          kubectl apply -f ./kerberos-vault-deployment.yaml -n kerberos-vault
-          kubectl apply -f ./kerberos-vault-service.yaml -n kerberos-vault
+          kubectl apply -f ./base/mongodb-configmap.yaml -n kerberos-vault
+          kubectl apply -f ./base/vault/kerberos-vault-deployment.yaml -n kerberos-vault
+          kubectl apply -f ./base/vault/kerberos-vault-service.yaml -n kerberos-vault
           echo "Sleeping for 30 seconds, give time for the helm chart to create the pods" && sleep 30
           kubectl get pods -A -o wide 
           kubectl get pods -A -o wide | grep kerberos-vault | awk '{print $3}' | grep -q '1/1' && echo "kerberos-vault pod is running with status 1/1" || (echo "kerberos-vault pod is not running with status 1/1" && exit 1)
       - name: Install Kerberos Agent
         id: install-kerberos-agent
         run: |
-          kubectl apply -f ./kerberos-agent-deployment.yaml
+          kubectl apply -f ./base/agent/kerberos-agent-deployment.yaml
           echo "Sleeping for 30 seconds, give time for the helm chart to create the pods" && sleep 30
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep agent | awk '{print $3}' | grep -q '1/1' && echo "kerberos-agent pod is running with status 1/1" || (echo "kerberos-agent pod is not running with status 1/1" && exit 1)
       - name: Install Data filtering
         id: install-data-filtering
         run: |
-          sed -e '/resources/ s/^#*/#/' -i ./data-filtering-deployment.yaml
-          sed -e '/limits/ s/^#*/#/' -i ./data-filtering-deployment.yaml
-          sed -e '/nvidia/ s/^#*/#/' -i ./data-filtering-deployment.yaml
-          kubectl apply -f data-filtering-deployment.yaml
+          sed -e '/resources/ s/^#*/#/' -i ./base/vault/data-filtering-deployment.yaml
+          sed -e '/limits/ s/^#*/#/' -i ./base/vault/data-filtering-deployment.yaml
+          sed -e '/nvidia/ s/^#*/#/' -i ./base/vault/data-filtering-deployment.yaml
+          kubectl apply -f ./base/vault/data-filtering-deployment.yaml
           echo "Sleeping for 250 seconds, give time for the helm chart to create the pods" && sleep 250
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep data-filtering | awk '{print $3}' | grep -q '1/1' && echo "data-filtering pod is running with status 1/1" || (echo "data-filtering pod is not running with status 1/1" && exit 1)

.github/workflows/microk8s.yaml

@@ -25,11 +25,16 @@ jobs:
           kubectl get pods -A -o wide
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: Install OpenEBS
+        run: |
+          kubectl apply -f https://openebs.github.io/charts/openebs-operator.yaml
+          echo "Sleeping for 60 seconds, give time for the operator to create the CRDs" && sleep 60
+          kubectl get sc
+          kubectl get po -A -o wide
       - name: Install Minio
         id: install-minio
         run: |
           git clone --depth 1 --branch v6.0.1 https://github.com/minio/operator.git && kubectl apply -k operator/
-          sed -i 's/openebs-hostpath/microk8s-hostpath/g' ./minio-tenant-base.yaml
           kubectl apply -f minio-tenant-base.yaml
           echo "Sleeping for 60 seconds, give time for the operator/tenant to create the CRDs" && sleep 60
           kubectl get po -A -o wide
@@ -39,9 +44,8 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           kubectl create namespace mongodb
-          sed -i 's/openebs-hostpath/microk8s-hostpath/g' ./mongodb-values.yaml
-          helm install mongodb -n mongodb bitnami/mongodb --values ./mongodb-values.yaml
-          echo "Sleeping for 90 seconds, give time for the helm chart to create the pods" && sleep 90
+          helm install mongodb -n mongodb bitnami/mongodb --values ./base/mongodb-values.yaml
+          echo "Sleeping for 250 seconds, give time for the helm chart to create the pods" && sleep 250
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep mongodb | awk '{print $3}' | grep -q '1/1' && echo "mongodb pod is running with status 1/1" || (echo "mongodb pod is not running with status 1/1" && exit 1)
       - name: Install RabbitMQ
@@ -49,8 +53,7 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           kubectl create namespace rabbitmq
-          sed -i 's/openebs-hostpath/microk8s-hostpath/g' ./rabbitmq-values.yaml
-          helm install rabbitmq -n rabbitmq bitnami/rabbitmq --values ./rabbitmq-values.yaml
+          helm install rabbitmq -n rabbitmq bitnami/rabbitmq --values ./base/rabbitmq-values.yaml
           echo "Sleeping for 60 seconds, give time for the helm chart to create the pods" && sleep 60
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep rabbitmq | awk '{print $3}' | grep -q '1/1' && echo "rabbitmq pod is running with status 1/1" || (echo "rabbitmq pod is not running with status 1/1" && exit 1)
@@ -58,26 +61,26 @@ jobs:
         id: install-kerberos-vault
         run: |
           kubectl create namespace kerberos-vault
-          kubectl apply -f ./mongodb-configmap.yaml -n kerberos-vault
-          kubectl apply -f ./kerberos-vault-deployment.yaml -n kerberos-vault
-          kubectl apply -f ./kerberos-vault-service.yaml -n kerberos-vault
+          kubectl apply -f ./base/mongodb-configmap.yaml -n kerberos-vault
+          kubectl apply -f ./base/vault/kerberos-vault-deployment.yaml -n kerberos-vault
+          kubectl apply -f ./base/vault/kerberos-vault-service.yaml -n kerberos-vault
           echo "Sleeping for 30 seconds, give time for the helm chart to create the pods" && sleep 30
           kubectl get pods -A -o wide 
           kubectl get pods -A -o wide | grep kerberos-vault | awk '{print $3}' | grep -q '1/1' && echo "kerberos-vault pod is running with status 1/1" || (echo "kerberos-vault pod is not running with status 1/1" && exit 1)
       - name: Install Kerberos Agent
         id: install-kerberos-agent
         run: |
-          kubectl apply -f ./kerberos-agent-deployment.yaml
+          kubectl apply -f ./base/agent/kerberos-agent-deployment.yaml
           echo "Sleeping for 30 seconds, give time for the helm chart to create the pods" && sleep 30
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep agent | awk '{print $3}' | grep -q '1/1' && echo "kerberos-agent pod is running with status 1/1" || (echo "kerberos-agent pod is not running with status 1/1" && exit 1)
       - name: Install Data filtering
         id: install-data-filtering
         run: |
-          sed -e '/resources/ s/^#*/#/' -i ./data-filtering-deployment.yaml
-          sed -e '/limits/ s/^#*/#/' -i ./data-filtering-deployment.yaml
-          sed -e '/nvidia/ s/^#*/#/' -i ./data-filtering-deployment.yaml
-          kubectl apply -f data-filtering-deployment.yaml
-          echo "Sleeping for 120 seconds, give time for the helm chart to create the pods" && sleep 120
+          sed -e '/resources/ s/^#*/#/' -i ./base/vault/data-filtering-deployment.yaml
+          sed -e '/limits/ s/^#*/#/' -i ./base/vault/data-filtering-deployment.yaml
+          sed -e '/nvidia/ s/^#*/#/' -i ./base/vault/data-filtering-deployment.yaml
+          kubectl apply -f ./base/vault/data-filtering-deployment.yaml
+          echo "Sleeping for 250 seconds, give time for the helm chart to create the pods" && sleep 250
           kubectl get pods -A -o wide
           kubectl get pods -A -o wide | grep data-filtering | awk '{print $3}' | grep -q '1/1' && echo "data-filtering pod is running with status 1/1" || (echo "data-filtering pod is not running with status 1/1" && exit 1)

base/hub/kerberos-hub-yolov8.yaml

@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: data-filtering
+  labels:
+    app: data-filtering
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: data-filtering
+  template:
+    metadata:
+      labels:
+        app: data-filtering
+    spec:
+      containers:
+        - name: data-filtering
+          image: uugai/data-filter:1.0.4
+          resources:
+            limits:
+              nvidia.com/gpu: 1 # requesting a single GPU
+          env:
+            - name: MODEL_NAME
+              value: "yolov8n.pt"
+            - name: CONDITION
+              value:  "1 persons detected"
+
+            - name: QUEUE_NAME
+              value: "data-filtering" # This is the topic of kafka we will read messages from.
+            - name: QUEUE_HOST
+              value: rabbitmq.rabbitmq:5672
+            - name: QUEUE_EXCHANGE
+            - name: QUEUE_USERNAME
+              value: yourusername
+            - name: QUEUE_PASSWORD
+              value: yourpassword
+
+            - name: STORAGE_URI
+              #value: http://vault-lb.kerberos-vault/api
+              value: "http://vault-nodeport.kerberos-vault:80/api"
+            - name: STORAGE_ACCESS_KEY
+              value: "XJoi2@bgSOvOYBy#"
+            - name: STORAGE_SECRET_KEY
+              value: "OGGqat4lXRpL@9XBYc8FUaId@5"
+
+            - name: LOGGING
+              value: "True"
+            - name: PLOT
+              value: "False"
+            - name: SAVE_VIDEO
+              value: "False"
+            - name: MEDIA_SAVEPATH
+              value: "/ml/data/input/video.mp4"
+            - name: OUTPUT_MEDIA_SAVEPATH
+              value: "/ml/data/input/output_video.mp4"
+
+            - name: TIME_VERBOSE
+              value: "True"
+
+            - name: CLASSIFICATION_FPS
+              value: "3"
+            - name: CLASSIFICATION_THRESHOLD
+              value: "0.3"
+            - name: MAX_NUMBER_OF_PREDICTIONS
+              value: "100"
+            - name: ALLOWED_CLASSIFICATIONS
+              value: "0, 1, 2, 3, 5, 7, 14, 15, 16, 24, 26, 28"
+
+            - name: FORWARDING_MEDIA
+              value: "True"
+            - name: REMOVE_AFTER_PROCESSED
+              value: "True"

base/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- mongodb-configmap.yaml
+- kerberos-factory-deployment.yaml
+- kerberos-factory-service.yaml
+- kerberos-factory-clusterrole.yaml
+- kerberos-vault-deployment.yaml
+- kerberos-vault-service.yaml

base/mongodb-configmap.yaml

@@ -6,6 +6,7 @@ data:
   # This is the mongodb database where data will be stored, you might use a different name if you want.
   MONGODB_DATABASE_STORAGE: "KerberosStorage"
   MONGODB_DATABASE_FACTORY: "KerberosFactory"
+  MONGODB_DATABASE_HUB: "Kerberos"
   # MongoDB URI (for example for a SaaS service like MongoDB Atlas)
   # If uri is set, the below properties are not used (host, adminDatabase, username, password)
   #MONGODB_URI: "mongodb+srv://xx:xx@kerberos-hub.xxx.mongodb.net/?retryWrites=true&w=majority&appName=xxx"

base/vault/data-filtering-deployment.yaml

@@ -18,8 +18,14 @@ spec:
         - name: data-filtering
           image: uugai/data-filter:1.0.4
           resources:
+            requests:
+              cpu: 100m
+              memory: 512Mi
+              nvidia.com/gpu: "1" 
             limits:
-              nvidia.com/gpu: 1 # requesting a single GPU
+              cpu: 100m
+              memory: 512Mi
+              nvidia.com/gpu: "1"  # requesting a single GPU
           env:
             - name: MODEL_NAME
               value: "yolov8n.pt"

kerberos-hub-yolov8.yaml

@@ -1,103 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: hub-yolov8
-  labels:
-    app: hub-yolov8
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: hub-yolov8
-  template:
-    metadata:
-      labels:
-        app: hub-yolov8
-    spec:
-      containers:
-        - name: hub-yolov8
-          image: uugai/hub-yolov8:latest
-          resources:
-            requests:
-              #nvidia.com/gpu: 1 # requesting a single GPU
-              cpu: 500m
-              memory: 1Gi
-            limits:
-              cpu: 500m
-              memory: 1Gi
-          env: 
-            # Queue parameters
-            - name: QUEUE_NAME
-              value: "kcloud-classify-queue.fifo" # This is the topic of RabbitMQ we will read messages from.
-            - name: TARGET_QUEUE_NAME
-              value: "kcloud-analysis-queue" # Once we processed the recording with ML, we will send results/metadata to a target topic of RabbitMQ.
-            - name: QUEUE_HOST
-              value: rabbitmq.rabbitmq:5672
-            - name: QUEUE_EXCHANGE
-            - name: QUEUE_USERNAME
-              value: yourusername
-            - name: QUEUE_PASSWORD
-              value: yourpassword
-
-            # Kerberos Vault parameters
-            - name: STORAGE_URI
-              value: "http://vault-nodeport.kerberos-vault:80/api"
-            - name: STORAGE_ACCESS_KEY
-              value: "XJoi2@bgSOvOYBy#"
-            - name: STORAGE_SECRET_KEY
-              value: "OGGqat4lXRpL@9XBYc8FUaId@5"
-
-            # YOLOv8 parameters
-            - name: MODEL_NAME
-              value: "yolov8n.pt"
-            - name: MEDIA_SAVEPATH
-              value: "/ml/data/input/video.mp4"
-
-            - name: LOGGING
-              value: "True"
-            - name: PLOT
-              value: "False"
-            - name: SAVE_VIDEO
-              value: "False"
-            - name: OUTPUT_MEDIA_SAVEPATH
-              value: "path/to/your/output_video.mp4"
-
-            - name: CREATE_BBOX_FRAME
-              value: "False"
-            - name: SAVE_BBOX_FRAME
-              value: "False"
-            - name: BBOX_FRAME_SAVEPATH
-              value: "path/to/your/output_bbox.jpg"
-
-            - name: CREATE_RETURN_JSON
-              value: "True"
-            - name: SAVE_RETURN_JSON
-              value: "False"
-            - name: RETURN_JSON_SAVEPATH
-              value: "path/to/your/output_json.json"
-
-            - name: TIME_VERBOSE
-              value: "True"
-
-            - name: CLASSIFICATION_FPS
-              value: "3"
-            - name: CLASSIFICATION_THRESHOLD
-              value: "0.3"
-            - name: MIN_DETECTIONS
-              value: "5"
-            - name: MIN_DISTANCE
-              value: "150"
-            - name: MIN_STATIC_DISTANCE
-              value: "0"
-            - name: MAX_NUMBER_OF_PREDICTIONS
-              value: "100"
-            - name: FIND_DOMINANT_COLORS
-              value: "False"
-            - name: ALLOWED_CLASSIFICATIONS
-              value: "0, 1, 2, 3, 5, 7, 14, 15, 16, 24, 26, 28"
-            - name: COLOR_PREDICTION_INTERVAL
-              value: "5"
-            - name: MIN_CLUSTERS
-              value: "4"
-            - name: MAX_CLUSTERS
-              value: "4"

overlays/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - standalone

overlays/standalone/kustomization.yaml

@@ -0,0 +1,22 @@
+metadata:
+  name: standalone
+resources:
+  - ../../base
+namePrefix: standalone-
+patches:
+  - target:
+      kind: Ingress
+      name: hub-api-ingress
+    patch: |-
+      - op: replace
+        path: /spec/tls/0/secretName
+        value: pr54-api
+      - op: replace
+        path: /spec/tls/0/hosts/0
+        value: pr54.api.kerberos.lol
+      - op: replace
+        path: /spec/rules/0/host
+        value: pr54.api.kerberos.lol
+      - op: replace
+        path: /spec/rules/0/http/paths/0/backend/service/name
+        value: pr54-hub-api-svc