From 6fd7a0e1aef17793d41a2362c0f8db53d9d83eac Mon Sep 17 00:00:00 2001 From: se-wo Date: Sun, 27 Mar 2022 20:11:10 +0200 Subject: [PATCH] Support CVAT 2.X deployment using helm (#4448) * Fix frontend helm labels taken. Previously those were erronously taken from backend. * Add support for CVAT 2.X in helm charts Added templates for OPA and extended readme on how to deploy. Moreover modernized the ingress chart to support k8s version 1.22+. * Update Changelog * Fix helm readme linter error Fix linter error due to markdown line longer than 120 characters --- CHANGELOG.md | 1 + helm-chart/Chart.yaml | 2 +- helm-chart/README.md | 12 ++- .../templates/cvat_frontend/deployment.yml | 6 +- helm-chart/templates/cvat_opa/config.yml | 17 ++++ helm-chart/templates/cvat_opa/deployment.yml | 82 +++++++++++++++++++ helm-chart/templates/cvat_opa/service.yml | 21 +++++ helm-chart/templates/ingress.yaml | 16 +++- helm-chart/values.yaml | 45 ++++++++++ 9 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 helm-chart/templates/cvat_opa/config.yml create mode 100644 helm-chart/templates/cvat_opa/deployment.yml create mode 100644 helm-chart/templates/cvat_opa/service.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 905d6cba..6df35fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Added missing geos dependency into Dockerfile () - Improved helm chart readme () +- Added helm chart support for CVAT 2.X and made ingress compatible with Kubernetes >=1.22 () ### Deprecated - TDB diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 567bb5e4..381331e6 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm-chart/README.md b/helm-chart/README.md index 867142a1..f4052e39 100644 --- a/helm-chart/README.md +++ b/helm-chart/README.md @@ -43,7 +43,11 @@ helm dependency update 1. Create `values.override.yaml` file inside `helm-chart` directory. 2. Fill `values.override.yaml` with new parameters for chart. 3. Override [postgresql password](#postgresql-password) -4. (Optional) Add [ingress parameters](#ingress-parameters) +4. Create a rules.tar.gz archive containing all OPA rules inside this `helm-chart` directory. + ```sh + find ../cvat/apps/iam/rules -name "*.rego" -and ! -name '*test*' -exec basename {} \; | tar -czf rules.tar.gz -C ../cvat/apps/iam/rules/ -T - + ``` +5. (Optional) Add [ingress parameters](#ingress-parameters) ### Postgresql password? Put below into your `values.override.yaml` @@ -99,6 +103,12 @@ ingress: ## Deployment Make sure you are using correct kubernetes context. You can check it with `kubectl config current-context`. +> **Warning:** The k8s service name of Open Policy Agent is fixed to opa by default. +This is done to be compatible with CVAT 2.0 but limits this helm chart to a single release per namespace. +The OPA url currently can´t be set as an environment variable. +As soon as this is possible you can set cvat.opa.composeCompatibleServiceName +to false in your value.override.yaml and configure the opa url as additional env. + Execute following command from repo root directory ### With overrides: ```helm upgrade -n -i --create-namespace ./helm-chart -f ./helm-chart/values.yaml -f ./helm-chart/values.override.yaml``` diff --git a/helm-chart/templates/cvat_frontend/deployment.yml b/helm-chart/templates/cvat_frontend/deployment.yml index 9274469e..a7b5648a 100644 --- a/helm-chart/templates/cvat_frontend/deployment.yml +++ b/helm-chart/templates/cvat_frontend/deployment.yml @@ -22,10 +22,10 @@ spec: {{- include "cvat.labels" . | nindent 8 }} app: cvat-app tier: frontend - {{- with .Values.cvat.backend.labels }} + {{- with .Values.cvat.frontend.labels }} {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.cvat.backend.annotations }} + {{- with .Values.cvat.frontend.annotations }} annotations: {{- toYaml . | nindent 8 }} {{- end }} @@ -62,4 +62,4 @@ spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} \ No newline at end of file diff --git a/helm-chart/templates/cvat_opa/config.yml b/helm-chart/templates/cvat_opa/config.yml new file mode 100644 index 00000000..e2903850 --- /dev/null +++ b/helm-chart/templates/cvat_opa/config.yml @@ -0,0 +1,17 @@ +{{- if .Values.cvat.opa.defaultStorage.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-opa-rules + namespace: {{ .Release.Namespace }} + labels: + {{- include "cvat.labels" . | nindent 4 }} + app: cvat-app + tier: opa +{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.GitVersion }} +immutable: true +{{- end }} +binaryData: + rules.tar.gz: |- + {{ .Files.Get "rules.tar.gz" | b64enc }} +{{- end}} \ No newline at end of file diff --git a/helm-chart/templates/cvat_opa/deployment.yml b/helm-chart/templates/cvat_opa/deployment.yml new file mode 100644 index 00000000..e1db2b0a --- /dev/null +++ b/helm-chart/templates/cvat_opa/deployment.yml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-opa + namespace: {{ .Release.Namespace }} + labels: + {{- include "cvat.labels" . | nindent 4 }} + app: cvat-app + tier: opa +spec: + replicas: {{ .Values.cvat.opa.replicas }} + strategy: + type: Recreate + selector: + matchLabels: + {{- include "cvat.labels" . | nindent 6 }} + app: cvat-app + tier: opa + template: + metadata: + labels: + {{- include "cvat.labels" . | nindent 8 }} + app: cvat-app + tier: opa + {{- with .Values.cvat.opa.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cvat.opa.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: cvat-opa-app-container + image: {{ .Values.cvat.opa.image }}:{{ .Values.cvat.opa.tag }} + args: + - run + - --server + - --addr + - :8181 + - --set=decision_logs.console=true + - /rules/rules.tar.gz + {{- with .Values.cvat.opa.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - containerPort: 8181 + {{- with .Values.cvat.opa.additionalEnv }} + env: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - mountPath: /rules + name: cvat-opa-rules + {{- with .Values.cvat.opa.additionalVolumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + {{- if .Values.cvat.opa.defaultStorage.enabled }} + - name: cvat-opa-rules + configMap: + name: "{{ .Release.Name }}-opa-rules" + items: + - key: "rules.tar.gz" + path: "rules.tar.gz" + {{- end }} + {{- with .Values.cvat.opa.additionalVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cvat.opa.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cvat.opa.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm-chart/templates/cvat_opa/service.yml b/helm-chart/templates/cvat_opa/service.yml new file mode 100644 index 00000000..add6430a --- /dev/null +++ b/helm-chart/templates/cvat_opa/service.yml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if .Values.cvat.opa.composeCompatibleServiceName }} + name: opa + {{- else }} + name: {{ .Release.Name }}-opa-service + {{- end }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cvat.labels" . | nindent 4 }} + app: cvat-app + tier: opa +spec: + selector: + app: cvat-app + tier: opa + {{- include "cvat.labels" . | nindent 4 }} + {{- with .Values.cvat.opa.service }} + {{- toYaml . | nindent 2 }} + {{- end }} diff --git a/helm-chart/templates/ingress.yaml b/helm-chart/templates/ingress.yaml index cab76c13..86fc4651 100644 --- a/helm-chart/templates/ingress.yaml +++ b/helm-chart/templates/ingress.yaml @@ -1,6 +1,9 @@ +{{- $version := .Capabilities.KubeVersion.GitVersion -}} {{- if .Values.ingress.enabled -}} {{- $fullName := include "cvat.fullname" . -}} -{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.19-0" $version -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" $version -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 @@ -16,6 +19,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: +{{- if semverCompare ">=1.19-0" $version }} + ingressClassName: nginx +{{- end }} {{- if .Values.ingress.tls }} tls: {{- range .Values.ingress.tls }} @@ -36,9 +42,17 @@ spec: {{- if .pathType }} pathType: {{ .pathType }} {{- end }} + {{- if semverCompare ">=1.19-0" $version }} + backend: + service: + name: {{ .service.name }} + port: + number: {{ .service.port }} + {{- else }} backend: serviceName: {{ .service.name }} servicePort: {{ .service.port }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index a4ed06c4..4bb3f264 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -99,6 +99,51 @@ cvat: targetPort: 80 protocol: TCP name: http + opa: + replicas: 1 + image: openpolicyagent/opa + tag: 0.34.2-rootless + labels: {} + # test: test + annotations: {} + # test.io/test: test + resources: {} + affinity: {} + tolerations: [] + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + additionalEnv: [] + # Example: + # - name: volume-from-secret + # - name: TEST + # value: "test" + additionalVolumes: [] + # Example(assumes that pvc was already created): + # - name: tmp + # persistentVolumeClaim: + # claimName: tmp + additionalVolumeMounts: [] + # Example: + # - mountPath: /tmp + # name: tmp + # subPath: test + composeCompatibleServiceName: true # Sets service name to opa in order to be docker-compose compatible. Necessary because changing IAM_OPA_DATA_URL via environment variables in current images. Hinders multiple deployment due to duplicate name + service: + type: ClusterIP + ports: + - port: 8181 + targetPort: 8181 + protocol: TCP + name: http + defaultStorage: + enabled: true postgresql: #See https://github.com/bitnami/charts/blob/master/bitnami/postgresql/ for more info