From 09a0d60360b51caa779743c981f072759c07d3dc Mon Sep 17 00:00:00 2001 From: 7va Date: Tue, 28 Oct 2025 13:59:14 -0400 Subject: [PATCH 01/37] add helm folder --- helm/.helmignore | 44 ++++++++++++++++++++++++++ helm/Chart.yaml | 7 ++++ helm/README.md | 17 ++++++++++ helm/templates/NOTES.txt | 9 ++++++ helm/templates/_helpers.tpl | 15 +++++++++ helm/templates/backend-deployment.yaml | 24 ++++++++++++++ helm/templates/backend-service.yaml | 14 ++++++++ helm/templates/db-service.yaml | 14 ++++++++ helm/templates/db-statefulset.yaml | 41 ++++++++++++++++++++++++ helm/templates/grafana-deployment.yaml | 26 +++++++++++++++ helm/templates/grafana-service.yaml | 14 ++++++++ helm/templates/ingress.yaml | 20 ++++++++++++ helm/templates/migrations-job.yaml | 21 ++++++++++++ helm/templates/pvc.yaml | 15 +++++++++ helm/values.yaml | 44 ++++++++++++++++++++++++++ 15 files changed, 325 insertions(+) create mode 100644 helm/.helmignore create mode 100644 helm/Chart.yaml create mode 100644 helm/README.md create mode 100644 helm/templates/NOTES.txt create mode 100644 helm/templates/_helpers.tpl create mode 100644 helm/templates/backend-deployment.yaml create mode 100644 helm/templates/backend-service.yaml create mode 100644 helm/templates/db-service.yaml create mode 100644 helm/templates/db-statefulset.yaml create mode 100644 helm/templates/grafana-deployment.yaml create mode 100644 helm/templates/grafana-service.yaml create mode 100644 helm/templates/ingress.yaml create mode 100644 helm/templates/migrations-job.yaml create mode 100644 helm/templates/pvc.yaml create mode 100644 helm/values.yaml diff --git a/helm/.helmignore b/helm/.helmignore new file mode 100644 index 0000000..5ea7d00 --- /dev/null +++ b/helm/.helmignore @@ -0,0 +1,44 @@ +# Common VCS directories +.git/ +.gitignore +.github/ + +# Mac / Windows system files +.DS_Store +Thumbs.db + +# Editor files +*.swp +*.bak +*.tmp +*.orig +*.log + +# Python / build artifacts +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +dist/ +build/ + +# Tests and CI/CD +tests/ +*.test +*.coverage +.coverage +.env +.vscode/ +.idea/ + +# Docs and misc +docs/ +*.md +LICENSE +CHANGELOG.md +README-PACKAGE.md + +# Docker and Compose files +docker-compose.yaml +Dockerfile +*.Dockerfile diff --git a/helm/Chart.yaml b/helm/Chart.yaml new file mode 100644 index 0000000..2abb772 --- /dev/null +++ b/helm/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: opensampl +description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) +type: application +version: 0.1.0 +appVersion: "1.0.0" +icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/README.md b/helm/README.md new file mode 100644 index 0000000..4f56c30 --- /dev/null +++ b/helm/README.md @@ -0,0 +1,17 @@ +charts/ +└── opensampl/ + ├── Chart.yaml + ├── values.yaml + ├── templates/ + │ ├── _helpers.tpl + │ ├── db-statefulset.yaml + │ ├── db-service.yaml + │ ├── grafana-deployment.yaml + │ ├── grafana-service.yaml + │ ├── backend-deployment.yaml + │ ├── backend-service.yaml + │ ├── migrations-job.yaml + │ ├── pvc.yaml + │ ├── ingress.yaml (optional) + │ └── NOTES.txt + └── .helmignore \ No newline at end of file diff --git a/helm/templates/NOTES.txt b/helm/templates/NOTES.txt new file mode 100644 index 0000000..44279c2 --- /dev/null +++ b/helm/templates/NOTES.txt @@ -0,0 +1,9 @@ +Thank you for installing OpenSAMPL! + +To access your services: +- Backend: ClusterIP service "{{ include "opensampl.name" . }}-backend" on port {{ .Values.backend.port }} +- Grafana: ClusterIP service "{{ include "opensampl.name" . }}-grafana" on port {{ .Values.grafana.port }} +- Database: StatefulSet "{{ include "opensampl.name" . }}-db" with {{ .Values.db.storage }} storage + +If ingress is enabled, access OpenSAMPL at: +http://{{ .Values.ingress.host }} diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl new file mode 100644 index 0000000..5ee2189 --- /dev/null +++ b/helm/templates/_helpers.tpl @@ -0,0 +1,15 @@ +{{/* Common naming helpers */}} +{{- define "opensampl.name" -}} +opensampl +{{- end -}} + + +{{- define "opensampl.fullname" -}} +{{ include "opensampl.name" . }}-{{ .Release.Name }} +{{- end -}} + + +{{- define "opensampl.labels" -}} +app.kubernetes.io/name: {{ include "opensampl.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/backend-deployment.yaml b/helm/templates/backend-deployment.yaml new file mode 100644 index 0000000..e0dc245 --- /dev/null +++ b/helm/templates/backend-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "opensampl.name" . }}-backend + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.backend.replicas }} + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-backend + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-backend + spec: + containers: + - name: backend + image: {{ .Values.backend.image }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.backend.port }} + env: + {{- toYaml .Values.backend.env | nindent 12 }} diff --git a/helm/templates/backend-service.yaml b/helm/templates/backend-service.yaml new file mode 100644 index 0000000..692a613 --- /dev/null +++ b/helm/templates/backend-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "opensampl.name" . }}-backend + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.backend.port }} + targetPort: {{ .Values.backend.port }} + name: http + selector: + app: {{ include "opensampl.name" . }}-backend diff --git a/helm/templates/db-service.yaml b/helm/templates/db-service.yaml new file mode 100644 index 0000000..b7a13c7 --- /dev/null +++ b/helm/templates/db-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "opensampl.name" . }}-db + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.db.port }} + targetPort: {{ .Values.db.port }} + name: postgres + selector: + app: {{ include "opensampl.name" . }}-db diff --git a/helm/templates/db-statefulset.yaml b/helm/templates/db-statefulset.yaml new file mode 100644 index 0000000..ed717f4 --- /dev/null +++ b/helm/templates/db-statefulset.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "opensampl.fullname" . }}-db +spec: + serviceName: {{ include "opensampl.fullname" . }}-db + replicas: 1 + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-db + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-db + spec: + containers: + - name: db + image: {{ .Values.db.image.repository }}:{{ .Values.db.image.tag }} + env: + - name: POSTGRES_DB + value: {{ .Values.db.database }} + - name: POSTGRES_USER + value: {{ .Values.db.user }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "opensampl.fullname" . }}-db-secret + key: password + ports: + - containerPort: 5432 + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.db.persistence.size }} diff --git a/helm/templates/grafana-deployment.yaml b/helm/templates/grafana-deployment.yaml new file mode 100644 index 0000000..3c6813e --- /dev/null +++ b/helm/templates/grafana-deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "opensampl.name" . }}-grafana + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-grafana + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-grafana + spec: + containers: + - name: grafana + image: {{ .Values.grafana.image }} + ports: + - containerPort: {{ .Values.grafana.port }} + env: + - name: GF_SECURITY_ADMIN_USER + value: {{ .Values.grafana.adminUser }} + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ .Values.grafana.adminPassword }} diff --git a/helm/templates/grafana-service.yaml b/helm/templates/grafana-service.yaml new file mode 100644 index 0000000..67a5d9f --- /dev/null +++ b/helm/templates/grafana-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "opensampl.name" . }}-grafana + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.grafana.port }} + targetPort: {{ .Values.grafana.port }} + name: http + selector: + app: {{ include "opensampl.name" . }}-grafana diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml new file mode 100644 index 0000000..25e1964 --- /dev/null +++ b/helm/templates/ingress.yaml @@ -0,0 +1,20 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "opensampl.name" . }}-ingress + annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.className }} +spec: + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "opensampl.name" . }}-backend + port: + number: {{ .Values.backend.port }} +{{- end }} diff --git a/helm/templates/migrations-job.yaml b/helm/templates/migrations-job.yaml new file mode 100644 index 0000000..f5fc367 --- /dev/null +++ b/helm/templates/migrations-job.yaml @@ -0,0 +1,21 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "opensampl.fullname" . }}-migrations + labels: + {{- include "opensampl.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade +spec: + template: + metadata: + labels: + app: {{ include "opensampl.name" . }} + spec: + restartPolicy: OnFailure + containers: + - name: migrations + image: {{ .Values.migrations.image }} + env: + - name: DATABASE_URL + values: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.name" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} \ No newline at end of file diff --git a/helm/templates/pvc.yaml b/helm/templates/pvc.yaml new file mode 100644 index 0000000..acfe1e6 --- /dev/null +++ b/helm/templates/pvc.yaml @@ -0,0 +1,15 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "opensampl.name" . }}-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} +{{- end }} diff --git a/helm/values.yaml b/helm/values.yaml new file mode 100644 index 0000000..e108869 --- /dev/null +++ b/helm/values.yaml @@ -0,0 +1,44 @@ +# Global settings +global: + imagePullPolicy: IfNotPresent + +# PostgreSQL Database +db: + image: postgres:14 + storage: 10Gi + username: opensampl + password: opensampl + database: opensampl + port: 5432 + +# Backend service +backend: + image: ghcr.io/ornl/opensampl-backend:latest + replicas: 1 + port: 8000 + env: + - name: DATABASE_URL + value: postgresql://opensampl:opensampl@opensampl-db:5432/opensampl + +# Grafana +grafana: + image: grafana/grafana:10.0.0 + port: 3000 + adminUser: admin + adminPassword: admin + +# Migrations +migrations: + image: ghcr.io/ornl/opensampl-migrations:latest + +# Persistence +persistence: + enabled: true + size: 10Gi + storageClass: "" + +# Ingress +ingress: + enabled: false + className: nginx + host: opensampl.local From d026a59f70373adc4f6b2442a55a53d16ad63c6e Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 11:07:47 -0400 Subject: [PATCH 02/37] add github actions workflow to build and push db image to ornl/opensampl --- .github/workflows/docker-publish.yml | 54 +++++++++++++++++++ .github/workflows/helm-chart.yaml | 33 ++++++++++++ helm/templates/_helpers.tpl | 21 ++++++-- helm/templates/backend-deployment.yaml | 24 --------- helm/templates/backend/deployment.yaml | 44 +++++++++++++++ .../service.yaml} | 10 ++-- helm/templates/db-service.yaml | 14 ----- helm/templates/db-statefulset.yaml | 41 -------------- helm/templates/db/configmap.yaml | 9 ++++ helm/templates/db/secret.yaml | 11 ++++ helm/templates/db/service.yaml | 16 ++++++ helm/templates/db/statefulset.yaml | 53 ++++++++++++++++++ helm/templates/grafana-deployment.yaml | 26 --------- helm/templates/grafana/deployment.yaml | 52 ++++++++++++++++++ .../service.yaml} | 10 ++-- helm/templates/ingress.yaml | 25 +++++++-- .../{ => migrations}/migrations-job.yaml | 3 +- helm/values.yaml | 49 +++++++++++------ opensampl/db/Dockerfile | 7 +++ opensampl/db/create-grafana.sh | 4 ++ 20 files changed, 368 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .github/workflows/helm-chart.yaml delete mode 100644 helm/templates/backend-deployment.yaml create mode 100644 helm/templates/backend/deployment.yaml rename helm/templates/{backend-service.yaml => backend/service.yaml} (68%) delete mode 100644 helm/templates/db-service.yaml delete mode 100644 helm/templates/db-statefulset.yaml create mode 100644 helm/templates/db/configmap.yaml create mode 100644 helm/templates/db/secret.yaml create mode 100644 helm/templates/db/service.yaml create mode 100644 helm/templates/db/statefulset.yaml delete mode 100644 helm/templates/grafana-deployment.yaml create mode 100644 helm/templates/grafana/deployment.yaml rename helm/templates/{grafana-service.yaml => grafana/service.yaml} (68%) rename helm/templates/{ => migrations}/migrations-job.yaml (67%) create mode 100644 opensampl/db/Dockerfile create mode 100644 opensampl/db/create-grafana.sh diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..e84ab2e --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,54 @@ +name: Build and Push DB Image + +on: + push: + branches: + - main + paths: + - 'opensampl/db/**' + - '.github/workflows/docker-publish-db.yml' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ornl/opensampl-db + +jobs: + build-and-push-db: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest + type=sha,prefix=main- + + - name: Build and push DB Docker image + uses: docker/build-push-action@v5 + with: + context: ./db + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/helm-chart.yaml b/.github/workflows/helm-chart.yaml new file mode 100644 index 0000000..d25bce9 --- /dev/null +++ b/.github/workflows/helm-chart.yaml @@ -0,0 +1,33 @@ +name: Helm Chart CI/CD + +on: + push: + branches: [ helm-deploy ] + pull_request: + branches: [ helm-deploy ] + workflow_dispatch: + +jobs: + helm: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Helm Lint + run: helm lint ./helm + + - name: Package Chart + run: helm package ./helm + + - name: Push to GHCR + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 5ee2189..6c3d8de 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -1,15 +1,28 @@ -{{/* Common naming helpers */}} +{{/* Base name */}} {{- define "opensampl.name" -}} opensampl {{- end -}} - +{{/* Full release-qualified name */}} {{- define "opensampl.fullname" -}} -{{ include "opensampl.name" . }}-{{ .Release.Name }} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" (include "opensampl.name" .) .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} {{- end -}} +{{/* Component-specific name */}} +{{- define "opensampl.componentname" -}} +{{- printf "%s-%s" (include "opensampl.name" .) .component | trunc 63 | trimSuffix "-" }} +{{- end -}} +{{/* Common labels */}} {{- define "opensampl.labels" -}} app.kubernetes.io/name: {{ include "opensampl.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} \ No newline at end of file +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .component }} +app.kubernetes.io/component: {{ .component }} +{{- end }} +{{- end -}} diff --git a/helm/templates/backend-deployment.yaml b/helm/templates/backend-deployment.yaml deleted file mode 100644 index e0dc245..0000000 --- a/helm/templates/backend-deployment.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "opensampl.name" . }}-backend - labels: - {{- include "opensampl.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.backend.replicas }} - selector: - matchLabels: - app: {{ include "opensampl.name" . }}-backend - template: - metadata: - labels: - app: {{ include "opensampl.name" . }}-backend - spec: - containers: - - name: backend - image: {{ .Values.backend.image }} - imagePullPolicy: {{ .Values.global.imagePullPolicy }} - ports: - - containerPort: {{ .Values.backend.port }} - env: - {{- toYaml .Values.backend.env | nindent 12 }} diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml new file mode 100644 index 0000000..7974e3d --- /dev/null +++ b/helm/templates/backend/deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "opensampl.fullname" . }}-backend + labels: + {{- include "opensampl.labels" . | nindent 4 }} + role: backend +spec: + replicas: {{ .Values.backend.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: backend + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: backend + spec: + initContainers: + - name: wait-for-db + image: busybox + command: + - sh + - -c + - > + until nc -z {{ include "opensampl.fullname" . }}-db {{ .Values.db.port }}; + do echo "Waiting for DB..."; + sleep 3; + done; + containers: + - name: backend + image: {{ .Values.backend.image }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.backend.port }} + env: + - name: DATABASE_URL + value: "postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }}" + {{- if .Values.backend.env }} + {{- toYaml .Values.backend.env | nindent 12 }} + {{- end }} diff --git a/helm/templates/backend-service.yaml b/helm/templates/backend/service.yaml similarity index 68% rename from helm/templates/backend-service.yaml rename to helm/templates/backend/service.yaml index 692a613..2f091c5 100644 --- a/helm/templates/backend-service.yaml +++ b/helm/templates/backend/service.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "opensampl.name" . }}-backend + name: {{ include "opensampl.fullname" . }}-backend labels: {{- include "opensampl.labels" . | nindent 4 }} spec: type: ClusterIP - ports: - - port: {{ .Values.backend.port }} - targetPort: {{ .Values.backend.port }} - name: http selector: app: {{ include "opensampl.name" . }}-backend + ports: + - name: http + port: {{ .Values.backend.port }} + targetPort: {{ .Values.backend.port }} diff --git a/helm/templates/db-service.yaml b/helm/templates/db-service.yaml deleted file mode 100644 index b7a13c7..0000000 --- a/helm/templates/db-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "opensampl.name" . }}-db - labels: - {{- include "opensampl.labels" . | nindent 4 }} -spec: - type: ClusterIP - ports: - - port: {{ .Values.db.port }} - targetPort: {{ .Values.db.port }} - name: postgres - selector: - app: {{ include "opensampl.name" . }}-db diff --git a/helm/templates/db-statefulset.yaml b/helm/templates/db-statefulset.yaml deleted file mode 100644 index ed717f4..0000000 --- a/helm/templates/db-statefulset.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "opensampl.fullname" . }}-db -spec: - serviceName: {{ include "opensampl.fullname" . }}-db - replicas: 1 - selector: - matchLabels: - app: {{ include "opensampl.name" . }}-db - template: - metadata: - labels: - app: {{ include "opensampl.name" . }}-db - spec: - containers: - - name: db - image: {{ .Values.db.image.repository }}:{{ .Values.db.image.tag }} - env: - - name: POSTGRES_DB - value: {{ .Values.db.database }} - - name: POSTGRES_USER - value: {{ .Values.db.user }} - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "opensampl.fullname" . }}-db-secret - key: password - ports: - - containerPort: 5432 - volumeMounts: - - mountPath: /var/lib/postgresql/data - name: data - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: {{ .Values.db.persistence.size }} diff --git a/helm/templates/db/configmap.yaml b/helm/templates/db/configmap.yaml new file mode 100644 index 0000000..17deed1 --- /dev/null +++ b/helm/templates/db/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "opensampl.fullname" . }}-db-config + labels: + {{- include "opensampl.labels" . | nindent 4 }} +data: + POSTGRES_DB: {{ .Values.db.database | quote }} + POSTGRES_PORT: "{{ .Values.db.port }}" diff --git a/helm/templates/db/secret.yaml b/helm/templates/db/secret.yaml new file mode 100644 index 0000000..79a49c5 --- /dev/null +++ b/helm/templates/db/secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "opensampl.fullname" . }}-db-secret + labels: + {{- include "opensampl.labels" . | nindent 4 }} +type: Opaque +stringData: + POSTGRES_USER: {{ .Values.db.username | quote }} + POSTGRES_PASSWORD: {{ .Values.db.password | quote }} + GF_SECURITY_ADMIN_PASSWORD: {{ .Values.db.grafanaPassword | quote }} diff --git a/helm/templates/db/service.yaml b/helm/templates/db/service.yaml new file mode 100644 index 0000000..6d3d185 --- /dev/null +++ b/helm/templates/db/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "opensampl.fullname" . }}-db + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: postgres + port: {{ .Values.db.port }} + targetPort: {{ .Values.db.port }} + selector: + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: db diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml new file mode 100644 index 0000000..27ab052 --- /dev/null +++ b/helm/templates/db/statefulset.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "opensampl.name" . }}-db + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + serviceName: {{ include "opensampl.name" . }}-db + replicas: 1 + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-db + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-db + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + containers: + - name: db + image: "{{ .Values.db.image.repository }}:{{ .Values.db.image.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.db.port }} + name: postgres + envFrom: + - secretRef: + name: {{ include "opensampl.fullname" . }}-db-secret + - configMapRef: + name: {{ include "opensampl.fullname" . }}-db-config + volumeMounts: + - name: db-storage + mountPath: /var/lib/postgresql/data + - name: init-script + mountPath: /docker-entrypoint-initdb.d/020_create_grafana_user.sh + subPath: 020_create_grafana_user.sh + volumes: + - name: init-script + configMap: + name: {{ include "opensampl.fullname" . }}-db-init + volumeClaimTemplates: + - metadata: + name: db-storage + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.db.persistence.size }} + {{- if .Values.db.persistence.storageClass }} + storageClassName: {{ .Values.db.persistence.storageClass }} + {{- end }} diff --git a/helm/templates/grafana-deployment.yaml b/helm/templates/grafana-deployment.yaml deleted file mode 100644 index 3c6813e..0000000 --- a/helm/templates/grafana-deployment.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "opensampl.name" . }}-grafana - labels: - {{- include "opensampl.labels" . | nindent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "opensampl.name" . }}-grafana - template: - metadata: - labels: - app: {{ include "opensampl.name" . }}-grafana - spec: - containers: - - name: grafana - image: {{ .Values.grafana.image }} - ports: - - containerPort: {{ .Values.grafana.port }} - env: - - name: GF_SECURITY_ADMIN_USER - value: {{ .Values.grafana.adminUser }} - - name: GF_SECURITY_ADMIN_PASSWORD - value: {{ .Values.grafana.adminPassword }} diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml new file mode 100644 index 0000000..5c3eca4 --- /dev/null +++ b/helm/templates/grafana/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "opensampl.fullname" . }}-grafana + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-grafana + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-grafana + spec: + containers: + - name: grafana + image: {{ .Values.grafana.image }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.grafana.port }} + env: + - name: GF_SECURITY_ADMIN_USER + value: {{ .Values.grafana.adminUser }} + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ .Values.grafana.adminPassword }} + - name: GF_DATABASE_TYPE + value: postgres + - name: GF_DATABASE_HOST + value: {{ include "opensampl.name" . }}-db:{{ .Values.db.port }} + - name: GF_DATABASE_NAME + value: {{ .Values.db.database }} + - name: GF_DATABASE_USER + value: {{ .Values.db.username }} + - name: GF_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "opensampl.fullname" . }}-db-secret + key: POSTGRES_PASSWORD + readinessProbe: + httpGet: + path: /login + port: {{ .Values.grafana.port }} + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /api/health + port: {{ .Values.grafana.port }} + initialDelaySeconds: 30 + periodSeconds: 10 diff --git a/helm/templates/grafana-service.yaml b/helm/templates/grafana/service.yaml similarity index 68% rename from helm/templates/grafana-service.yaml rename to helm/templates/grafana/service.yaml index 67a5d9f..6070f7f 100644 --- a/helm/templates/grafana-service.yaml +++ b/helm/templates/grafana/service.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "opensampl.name" . }}-grafana + name: {{ include "opensampl.fullname" . }}-grafana labels: {{- include "opensampl.labels" . | nindent 4 }} spec: type: ClusterIP - ports: - - port: {{ .Values.grafana.port }} - targetPort: {{ .Values.grafana.port }} - name: http selector: app: {{ include "opensampl.name" . }}-grafana + ports: + - name: http + port: {{ .Values.grafana.port }} + targetPort: {{ .Values.grafana.port }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 25e1964..40bdfa4 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -2,19 +2,38 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "opensampl.name" . }}-ingress + name: {{ include "opensampl.fullname" . }}-ingress + labels: + {{- include "opensampl.labels" . | nindent 4 }} annotations: kubernetes.io/ingress.class: {{ .Values.ingress.className }} + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ include "opensampl.fullname" . }}-tls rules: - host: {{ .Values.ingress.host }} http: paths: - - path: / + # Backend API + - path: /api pathType: Prefix backend: service: - name: {{ include "opensampl.name" . }}-backend + name: {{ include "opensampl.fullname" . }}-backend port: number: {{ .Values.backend.port }} + + # Grafana UI + - path: /grafana + pathType: Prefix + backend: + service: + name: {{ include "opensampl.fullname" . }}-grafana + port: + number: {{ .Values.grafana.port }} {{- end }} diff --git a/helm/templates/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml similarity index 67% rename from helm/templates/migrations-job.yaml rename to helm/templates/migrations/migrations-job.yaml index f5fc367..0fb36e8 100644 --- a/helm/templates/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -6,6 +6,7 @@ metadata: {{- include "opensampl.labels" . | nindent 4 }} annotations: "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: template: metadata: @@ -18,4 +19,4 @@ spec: image: {{ .Values.migrations.image }} env: - name: DATABASE_URL - values: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.name" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} \ No newline at end of file + value: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.name" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index e108869..276f8fb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -4,40 +4,59 @@ global: # PostgreSQL Database db: - image: postgres:14 - storage: 10Gi + image: + #repository: ghcr.io/mzinternallab/opensampl-dbc + repository: ghcr.io/ornl/opensampl-db + tag: latest username: opensampl password: opensampl + grafanaPassword: grafana123 database: opensampl port: 5432 + storage: 10Gi + persistence: + enabled: true + size: 10Gi + storageClass: "" # Backend service backend: - image: ghcr.io/ornl/opensampl-backend:latest + image: + repository: ghcr.io/ornl/opensampl-backend + tag: latest replicas: 1 port: 8000 env: - - name: DATABASE_URL - value: postgresql://opensampl:opensampl@opensampl-db:5432/opensampl + # Additional backend environment variables (merged automatically) + LOG_LEVEL: INFO + USE_API_KEY: "false" -# Grafana +# Grafana service grafana: - image: grafana/grafana:10.0.0 + image: + repository: grafana/grafana + tag: "10.0.0" + replicas: 1 port: 3000 adminUser: admin adminPassword: admin + database: + enabled: true + type: postgres + host: opensampl-db + port: 5432 + name: opensampl + user: opensampl -# Migrations +# Migrations (optional) migrations: - image: ghcr.io/ornl/opensampl-migrations:latest - -# Persistence -persistence: enabled: true - size: 10Gi - storageClass: "" + image: + repository: ghcr.io/ornl/opensampl-migrations + tag: latest + env: {} -# Ingress +# Ingress (optional) ingress: enabled: false className: nginx diff --git a/opensampl/db/Dockerfile b/opensampl/db/Dockerfile new file mode 100644 index 0000000..a16469b --- /dev/null +++ b/opensampl/db/Dockerfile @@ -0,0 +1,7 @@ +FROM timescale/timescaledb-ha:pg16.2-ts2.14.2-all + +ENV DEBIAN_FRONTEND=noninteractive + +COPY create-grafana.sh /docker-entrypoint-initdb.d/020_create_grafana_user.sh + +CMD ["postgres"] \ No newline at end of file diff --git a/opensampl/db/create-grafana.sh b/opensampl/db/create-grafana.sh new file mode 100644 index 0000000..daa9694 --- /dev/null +++ b/opensampl/db/create-grafana.sh @@ -0,0 +1,4 @@ +#!/bin/bash +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" < Date: Thu, 30 Oct 2025 12:09:54 -0400 Subject: [PATCH 03/37] build and push db image to ghcr/ornl/opensampl --- .github/workflows/docker-publish.yml | 54 ---------------------- .github/workflows/helm-chart.yaml | 33 -------------- .github/workflows/helm-chart.yml | 62 ++++++++++++++++++++++++++ helm/templates/backend/deployment.yaml | 4 +- helm/templates/db/pvc.yaml | 15 +++++++ helm/templates/pvc.yaml | 15 ------- helm/values.yaml | 8 ++-- 7 files changed, 84 insertions(+), 107 deletions(-) delete mode 100644 .github/workflows/docker-publish.yml delete mode 100644 .github/workflows/helm-chart.yaml create mode 100644 .github/workflows/helm-chart.yml create mode 100644 helm/templates/db/pvc.yaml delete mode 100644 helm/templates/pvc.yaml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index e84ab2e..0000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Build and Push DB Image - -on: - push: - branches: - - main - paths: - - 'opensampl/db/**' - - '.github/workflows/docker-publish-db.yml' - workflow_dispatch: - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ornl/opensampl-db - -jobs: - build-and-push-db: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=latest - type=sha,prefix=main- - - - name: Build and push DB Docker image - uses: docker/build-push-action@v5 - with: - context: ./db - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/helm-chart.yaml b/.github/workflows/helm-chart.yaml deleted file mode 100644 index d25bce9..0000000 --- a/.github/workflows/helm-chart.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Helm Chart CI/CD - -on: - push: - branches: [ helm-deploy ] - pull_request: - branches: [ helm-deploy ] - workflow_dispatch: - -jobs: - helm: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Helm - uses: azure/setup-helm@v3 - with: - version: v3.14.0 - - - name: Helm Lint - run: helm lint ./helm - - - name: Package Chart - run: helm package ./helm - - - name: Push to GHCR - env: - CR_PAT: ${{ secrets.GITHUB_TOKEN }} - run: | - helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml new file mode 100644 index 0000000..61ec37c --- /dev/null +++ b/.github/workflows/helm-chart.yml @@ -0,0 +1,62 @@ +name: Helm Chart CI/CD + +on: + push: + branches: [ helm-deploy ] + pull_request: + branches: [ helm-deploy ] + workflow_dispatch: + +jobs: + build-docker-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push DB image + uses: docker/build-push-action@v5 + with: + context: ./db + push: true + tags: ghcr.io/ornl/opensampl-db:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + helm: + runs-on: ubuntu-latest + needs: build-docker-images + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Helm Lint + run: helm lint ./helm + + - name: Package Chart + run: helm package ./helm + + - name: Push to GHCR + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts \ No newline at end of file diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index 7974e3d..ef2dd75 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -39,6 +39,6 @@ spec: env: - name: DATABASE_URL value: "postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }}" - {{- if .Values.backend.env }} - {{- toYaml .Values.backend.env | nindent 12 }} + {{- with .Values.backend.env }} + {{- toYaml . | nindent 12 }} {{- end }} diff --git a/helm/templates/db/pvc.yaml b/helm/templates/db/pvc.yaml new file mode 100644 index 0000000..b049791 --- /dev/null +++ b/helm/templates/db/pvc.yaml @@ -0,0 +1,15 @@ +{{- if .Values.db.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "opensampl.name" . }}-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.db.persistence.size }} + {{- if .Values.db.persistence.storageClass }} + storageClassName: {{ .Values.db.persistence.storageClass }} + {{- end }} +{{- end }} diff --git a/helm/templates/pvc.yaml b/helm/templates/pvc.yaml deleted file mode 100644 index acfe1e6..0000000 --- a/helm/templates/pvc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "opensampl.name" . }}-pvc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.persistence.size }} - {{- if .Values.persistence.storageClass }} - storageClassName: {{ .Values.persistence.storageClass }} - {{- end }} -{{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 276f8fb..8fcede9 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -27,9 +27,11 @@ backend: replicas: 1 port: 8000 env: - # Additional backend environment variables (merged automatically) - LOG_LEVEL: INFO - USE_API_KEY: "false" + # Additional backend environment variables + - name: LOG_LEVEL + value: INFO + - name: USE_API_KEY + value: "false" # Grafana service grafana: From 5fa3c989d31c5d24d50c6e94c05367e9d072daa8 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 12:15:55 -0400 Subject: [PATCH 04/37] use workflow to build and push db image to ghcr.io.ornl/opensampl --- .github/workflows/helm-chart.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 61ec37c..9b15183 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -30,7 +30,7 @@ jobs: - name: Build and push DB image uses: docker/build-push-action@v5 with: - context: ./db + context: ./opensampl/db push: true tags: ghcr.io/ornl/opensampl-db:latest cache-from: type=gha From d3b01d7ffc2063eddaa5d63dd5f01d9597ab224b Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 13:33:38 -0400 Subject: [PATCH 05/37] build and push backend and grafana to ghcr for testing helm deployment - minus migratins --- .github/workflows/helm-chart.yml | 13 ++ helm/values.yaml | 1 - opensampl/backend/Dockerfile | 9 + opensampl/backend/README.d | 2 + opensampl/backend/main.py | 353 +++++++++++++++++++++++++++++ opensampl/backend/requirements.txt | 8 + 6 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 opensampl/backend/Dockerfile create mode 100644 opensampl/backend/README.d create mode 100644 opensampl/backend/main.py create mode 100644 opensampl/backend/requirements.txt diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 9b15183..5b6db77 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -13,6 +13,19 @@ jobs: permissions: contents: read packages: write + + strategy: + matrix: + service: + - name: db + context: ./opensampl/db + - name: backend + context: ./opensampl/backend + - name: grafana + context: ./opensampl/server/grafana + #- name: migrations + # context: ./opensampl/migrations + steps: - name: Checkout uses: actions/checkout@v4 diff --git a/helm/values.yaml b/helm/values.yaml index 8fcede9..77f7ed0 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -5,7 +5,6 @@ global: # PostgreSQL Database db: image: - #repository: ghcr.io/mzinternallab/opensampl-dbc repository: ghcr.io/ornl/opensampl-db tag: latest username: opensampl diff --git a/opensampl/backend/Dockerfile b/opensampl/backend/Dockerfile new file mode 100644 index 0000000..d1ab545 --- /dev/null +++ b/opensampl/backend/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.11 +WORKDIR /tmp +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && \ + rm requirements.txt +ENV ROUTE_TO_BACKEND=false +COPY main.py . +CMD ["uvicorn", "main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8000"] +# CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/opensampl/backend/README.d b/opensampl/backend/README.d new file mode 100644 index 0000000..f46eed8 --- /dev/null +++ b/opensampl/backend/README.d @@ -0,0 +1,2 @@ +CAST Backend +http://localhost:8000/docs - swagger endpoint \ No newline at end of file diff --git a/opensampl/backend/main.py b/opensampl/backend/main.py new file mode 100644 index 0000000..26fbb7c --- /dev/null +++ b/opensampl/backend/main.py @@ -0,0 +1,353 @@ +import io +import json +import os +import sys +import time +from typing import Any, Dict, Callable, Optional +from datetime import datetime, timedelta +import pandas as pd +from fastapi import FastAPI, HTTPException, File, UploadFile, Form, Request, Response, Security, Depends +from fastapi.security.api_key import APIKeyHeader +from fastapi.responses import JSONResponse, RedirectResponse +from loguru import logger +from pydantic import BaseModel +from sqlalchemy import create_engine, text, select, or_, and_ +from sqlalchemy.exc import SQLAlchemyError, IntegrityError +from sqlalchemy.orm import sessionmaker, Session +from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST +from opensampl.db.access_orm import APIAccessKey +from opensampl import load_data +from opensampl.db.orm import ProbeMetadata +from opensampl.vendors.constants import VendorType, ProbeKey +from opensampl.metrics import METRICS, MetricType +from opensampl.references import REF_TYPES, CompoundReferenceType, ReferenceType +import psycopg2 + +class TimeDataPoint(BaseModel): + time: str + value: float + + +class WriteTablePayload(BaseModel): + table: str + data: Dict[str, Any] + if_exists: load_data.conflict_actions = 'update' + + +class ProbeMetadataPayload(BaseModel): + vendor: VendorType + probe_key: ProbeKey + data: Dict[str, Any] + + +DATABASE_URI = os.getenv("DATABASE_URL") +engine = create_engine(DATABASE_URI) + +loglevel = os.getenv("BACKEND_LOG_LEVEL", "INFO") +app = FastAPI() + + +REQUEST_COUNT = Counter( + "http_requests_total", + "Total number of HTTP requests", + ["method", "endpoint", "http_status"] +) + +REQUEST_LATENCY = Histogram( + "http_request_duration_seconds", + "Duration of HTTP requests in seconds", + ["method", "endpoint"] +) + +EXCLUDED_PATHS = {"/metrics", "/healthcheck", "/healthcheck_database", "/healthcheck_metadata"} + +logger.configure(handlers=[{"sink": sys.stderr, "level": loglevel}]) + +USE_API_KEY = os.getenv("USE_API_KEY", "false").lower() == "true" +API_KEY_NAME = "access-key" + +api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) + +def get_keys(): + env_keys = os.getenv('API_KEYS', '').strip() + keys = [k.strip() for k in env_keys.split(',') if k.strip()] + if keys: + logger.debug("api access keys loaded from env") + return keys + try: + Session = sessionmaker(bind=engine) + with Session() as session: + now = datetime.utcnow() + stmt = select(APIAccessKey.key).where( + or_( + APIAccessKey.expires_at == None, + APIAccessKey.expires_at > now + ) + ) + result = session.execute(stmt) + keys = [row[0] for row in result.all()] + logger.debug("api access keys loaded from db") + return keys + except Exception as e: + logger.debug(f"exception attempting to load api access keys from db: {e}") + return [] + +def validate_api_key(api_key: str = Security(api_key_header)): + if not USE_API_KEY: + return # Security is disabled + if api_key not in get_keys(): + raise HTTPException(status_code=403, detail="Invalid or missing API key") + return api_key + +def get_db(): + Session = sessionmaker(bind=engine) + try: + session = Session() + yield session + finally: + session.close() + +@app.middleware("http") +async def metrics_middleware(request: Request, call_next: Callable) -> Response: + """Middleware to track request metrics.""" + if request.url.path in EXCLUDED_PATHS: + return await call_next(request) + start_time = time.time() + response: Response = await call_next(request) + duration = time.time() - start_time + + REQUEST_COUNT.labels( + method=request.method, + endpoint=request.url.path, + http_status=response.status_code + ).inc() + + REQUEST_LATENCY.labels( + method=request.method, + endpoint=request.url.path + ).observe(duration) + + return response + + +# add route to docs from / to /docs +@app.get("/", include_in_schema=False) +async def docs_redirect(): + return RedirectResponse(url='/docs') + + +@app.get("/setloglevel") +def set_log_level(newloglevel: str, api_key: str = Depends(validate_api_key)): + """ + change visible log level in backend container + """ + newloglevel = newloglevel.upper() + logger.configure(handlers=[{"sink": sys.stderr, "level": newloglevel}]) + return {"loglevel": newloglevel} + + +@app.get("/checkloglevel") +def check_log_level(api_key: str = Depends(validate_api_key)): + """ + This is to check which log levels are visible in backend container + """ + logger.debug('Debug test') + logger.info('Info test') + logger.warning('Warning test') + logger.error('Error test') + current_level = list(logger._core.handlers.values())[0]["level"].name + return {"loglevel": current_level} + + +@app.post("/write_to_table") +def write_to_table(payload: WriteTablePayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): + try: + load_data.write_to_table(table=payload.table, data=payload.data, if_exists=payload.if_exists, session=session) + logger.debug(f'Successfully wrote to {payload.table} using: {payload.data}') + return JSONResponse(content={"message": f"Succeeded loading data into {payload.table}"}, status_code=200) + except IntegrityError as e: + if isinstance(e.orig, psycopg2.errors.UniqueViolation): + return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) + return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) + except SQLAlchemyError as e: + logger.error(f'SQLAlchemy error: {e}') + return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) + except json.JSONDecodeError as e: + logger.error(f'JSON decode error: {e}') + return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) + except Exception as e: + logger.error(f'Unexpected error: {e}') + return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) + + +@app.post("/load_time_data") +async def load_time_data(probe_key_str: str = Form(...), + metric_type_str: Optional[str] = Form(None), + reference_type_str: Optional[str] = Form(None), + compound_key_str: Optional[str] = Form(None), + file: UploadFile = File(...), + api_key: str = Depends(validate_api_key), + session: Session = Depends(get_db)): + try: + + probe_key = ProbeKey(**json.loads(probe_key_str)) + + if metric_type_str is not None: + metric_type_dict = json.loads(metric_type_str) + metric_type = MetricType(**metric_type_dict) + else: + metric_type = METRICS.UNKNOWN + + if reference_type_str is not None: + reference_type_dict = json.loads(reference_type_str) + if 'reference_table' in reference_type_dict: + reference_type = CompoundReferenceType(**reference_type_dict) + else: + reference_type = ReferenceType(**reference_type_dict) + else: + reference_type = REF_TYPES.UNKNOWN + + compound_key = None if compound_key_str is None else json.loads(compound_key_str) + + content = await file.read() + df = pd.read_csv(io.BytesIO(content)) + logger.info(df.head()) + # Convert time strings back to datetime + df['time'] = pd.to_datetime(df['time']) + + # Convert value strings back to float64 + # df['value'] = df['value'].astype('float64') + + # Use the same load_time_data function as before + load_data.load_time_data( + probe_key=probe_key, + metric_type=metric_type, + reference_type=reference_type, + compound_key=compound_key, + data=df, + session=session + ) + + return JSONResponse( + content={"message": f"Successfully loaded {len(df)} data points"}, + status_code=200 + ) + except IntegrityError as e: + if session: + session.rollback() + session.close() + if isinstance(e.orig, psycopg2.errors.UniqueViolation): + return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) + return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) + except SQLAlchemyError as e: + logger.error(f'Database error: {e}') + if session: + session.rollback() + session.close() + raise HTTPException(status_code=500, detail=f"Database error: {str(e)}") + except Exception as e: + logger.error(f'Unexpected error: {e}') + if session: + session.rollback() + session.close() + raise HTTPException(status_code=500, detail=f"Error processing time series data: {str(e)}") + + +@app.post("/load_probe_metadata") +def load_probe_metadata(payload: ProbeMetadataPayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): + logger.debug(f"Received payload: {payload.model_dump()}") + + try: + load_data.load_probe_metadata(vendor=payload.vendor, probe_key=payload.probe_key, data=payload.data, + session=session) + logger.debug( + f'Successfully wrote to {ProbeMetadata.__tablename__} and {payload.vendor.metadata_table}: {payload.data}') + return JSONResponse(content={"message": f"Succeeded loaded metadata for {payload.probe_key}"}, status_code=200) + except IntegrityError as e: + session.rollback() + if isinstance(e.orig, psycopg2.errors.UniqueViolation): + return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) + return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) + except SQLAlchemyError as e: + logger.error(f'SQLAlchemy error: {e}') + return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) + except json.JSONDecodeError as e: + logger.error(f'JSON decode error: {e}') + return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) + except Exception as e: + logger.exception(f'Unexpected error: {e}') + return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) + + +@app.get("/create_new_tables") +def create_new_tables(create_schema: bool = True, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): + try: + load_data.create_new_tables(create_schema=create_schema, session=session) + return JSONResponse(content={"message": f"Succeeded in creating any new tables"}, status_code=200) + except SQLAlchemyError as e: + logger.error(f'SQLAlchemy error: {e}') + return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) + except json.JSONDecodeError as e: + logger.error(f'JSON decode error: {e}') + return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) + except Exception as e: + logger.error(f'Unexpected error: {e}') + return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) + + +@app.get("/gen_api_key") +def generate_api_key(expire_after: Optional[int] = None, session: Session = Depends(get_db)): + try: + + new_key = APIAccessKey() + new_key.generate_key() + if expire_after: + new_key.expires_at = datetime.utcnow() + timedelta(days=expire_after) + + session.add(new_key) + + session.commit() + return JSONResponse(content={"message": f"Succeeded in creating new access key"}, status_code=200) + except SQLAlchemyError as e: + logger.error(f'SQLAlchemy error: {e}') + return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) + except Exception as e: + logger.error(f'Unexpected error: {e}') + return JSONResponse(content={"message": f"Failed to create new access key: {e}"}, status_code=500) + + +@app.get("/healthcheck") +def healthcheck(): + return {"status": "OK"} + +@app.get("/healthcheck_database") +def healthcheck_db(): + try: + with engine.connect() as connection: + connection.execute(text("SELECT 1")) + return {"status": "OK"} + except SQLAlchemyError: + raise HTTPException(status_code=503, detail="Database connection error") + + +@app.get("/healthcheck_metadata") +def healthcheck_metadata(): + try: + with engine.connect() as connection: + result = connection.execute( + text("SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'castdb';")) + schema_exists = result.fetchone() is not None + if schema_exists: + return {"status": "OK"} + else: + raise HTTPException(status_code=500, detail="Schema 'castdb' does not exist") + except SQLAlchemyError as e: + raise HTTPException(status_code=503, detail=f"Database connection error: {str(e)}") + + +@app.get("/metrics", include_in_schema=False) +def metrics(): + """ + Expose Prometheus metrics. + """ + return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) diff --git a/opensampl/backend/requirements.txt b/opensampl/backend/requirements.txt new file mode 100644 index 0000000..ca91da0 --- /dev/null +++ b/opensampl/backend/requirements.txt @@ -0,0 +1,8 @@ +psycopg2-binary +fastapi +uvicorn +sqlalchemy +loguru +python-multipart +prometheus-client +opensampl==1.1.5 From 39889ed1eb12d816ecfc333041502fda0937b760 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 13:50:48 -0400 Subject: [PATCH 06/37] build backend and grafana to test helm deployment --- .github/workflows/helm-chart.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 5b6db77..42e325e 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -72,4 +72,6 @@ jobs: CR_PAT: ${{ secrets.GITHUB_TOKEN }} run: | helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts \ No newline at end of file + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts + + \ No newline at end of file From 86db46acae1c3d88f35206f49ca4ee95b20bbb11 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 13:55:39 -0400 Subject: [PATCH 07/37] corrected the build and push phase to use the matrix build for all images --- .github/workflows/helm-chart.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 42e325e..6e234c0 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -40,14 +40,14 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push DB image + - name: Build and push ${{ matrix.service.name }} image uses: docker/build-push-action@v5 with: - context: ./opensampl/db + context: ${{ matrix.service.context }} push: true - tags: ghcr.io/ornl/opensampl-db:latest - cache-from: type=gha - cache-to: type=gha,mode=max + tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest + cache-from: type=gha,scope=${{ matrix.service.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} helm: runs-on: ubuntu-latest @@ -74,4 +74,3 @@ jobs: helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT helm push opensampl-*.tgz oci://ghcr.io/ornl/charts - \ No newline at end of file From e8f24272ba3ba470df8cdd669a9538435115a9a9 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 14:44:19 -0400 Subject: [PATCH 08/37] testing helm deploy and minikube test --- .github/workflows/helm-chart.old | 76 ++++++++++++++++++++++++++++++++ .github/workflows/helm-chart.yml | 69 ++++++++++++++++++++++++++--- .github/workflows/helm-readme.md | 27 ++++++++++++ helm/values.yaml | 4 +- 4 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/helm-chart.old create mode 100644 .github/workflows/helm-readme.md diff --git a/.github/workflows/helm-chart.old b/.github/workflows/helm-chart.old new file mode 100644 index 0000000..6e234c0 --- /dev/null +++ b/.github/workflows/helm-chart.old @@ -0,0 +1,76 @@ +name: Helm Chart CI/CD + +on: + push: + branches: [ helm-deploy ] + pull_request: + branches: [ helm-deploy ] + workflow_dispatch: + +jobs: + build-docker-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + service: + - name: db + context: ./opensampl/db + - name: backend + context: ./opensampl/backend + - name: grafana + context: ./opensampl/server/grafana + #- name: migrations + # context: ./opensampl/migrations + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push ${{ matrix.service.name }} image + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.service.context }} + push: true + tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest + cache-from: type=gha,scope=${{ matrix.service.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} + + helm: + runs-on: ubuntu-latest + needs: build-docker-images + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Helm Lint + run: helm lint ./helm + + - name: Package Chart + run: helm package ./helm + + - name: Push to GHCR + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts + diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 6e234c0..889a967 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -40,24 +40,84 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/ornl/opensampl-${{ matrix.service.name }} + tags: | + type=ref,event=pr + type=ref,event=branch + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} + - name: Build and push ${{ matrix.service.name }} image uses: docker/build-push-action@v5 with: context: ${{ matrix.service.context }} push: true - tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha,scope=${{ matrix.service.name }} cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} - helm: + test-helm-chart: + name: Test Helm Chart Deployment runs-on: ubuntu-latest needs: build-docker-images + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start minikube + uses: medyagh/setup-minikube@latest + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Install Helm Chart + working-directory: helm/ + run: | + PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}') + + helm install test-opensampl . \ + --set db.image.tag=pr-${PR_NUMBER} \ + --set backend.image.tag=pr-${PR_NUMBER} \ + --set grafana.image.tag=pr-${PR_NUMBER} \ + --wait \ + --timeout 5m + + - name: Verify Deployment + run: | + kubectl get pods + kubectl get services + kubectl get pvc + + - name: Show Pod Logs on Failure + if: failure() + run: | + echo "=== Pod Status ===" + kubectl get pods + echo "=== Pod Descriptions ===" + kubectl describe pods + echo "=== Pod Logs ===" + for pod in $(kubectl get pods -o name); do + echo "Logs for $pod:" + kubectl logs $pod --all-containers=true || true + done + + helm-package: + name: Package and Push Helm Chart + runs-on: ubuntu-latest + needs: build-docker-images + if: github.event_name != 'pull_request' steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Helm - uses: azure/setup-helm@v3 + uses: azure/setup-helm@v4 with: version: v3.14.0 @@ -72,5 +132,4 @@ jobs: CR_PAT: ${{ secrets.GITHUB_TOKEN }} run: | helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts - + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts \ No newline at end of file diff --git a/.github/workflows/helm-readme.md b/.github/workflows/helm-readme.md new file mode 100644 index 0000000..4580128 --- /dev/null +++ b/.github/workflows/helm-readme.md @@ -0,0 +1,27 @@ +### 1. **Smart Image Tagging** +Images now get tagged with: +- `pr-123` for pull requests +- `helm-deploy-abc123` for branch commits +- `latest` for main branch + +### 2. **New Test Job** (`test-helm-chart`) +- **Only runs on PRs** (not regular pushes) +- Spins up local Kubernetes (minikube) +- Installs your Helm chart with PR-tagged images +- Verifies pods start successfully +- Shows detailed logs if anything fails + +### 3. **Separated Helm Packaging** +- `helm-package` job only runs on **direct pushes** (not PRs) +- Lints, packages, and pushes the chart + +## Workflow Flow + +**On Pull Request:** +``` +Build Images (with pr-123 tags) → Test in Kubernetes → ✓ Pass/Fail +``` + +**On Push to helm-deploy:** +``` +Build Images (with latest tag) → Package & Push Helm Chart \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 77f7ed0..dbae2f6 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -35,8 +35,8 @@ backend: # Grafana service grafana: image: - repository: grafana/grafana - tag: "10.0.0" + repository: ghcr.io/ornl/opensampl-grafana + tag: latest replicas: 1 port: 3000 adminUser: admin From f855e019264bfec19ec2d13753e9c54064aeca15 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 14:59:22 -0400 Subject: [PATCH 09/37] test helm deploy and minikube testing --- .github/workflows/helm-chart.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 889a967..38e52c7 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -4,7 +4,9 @@ on: push: branches: [ helm-deploy ] pull_request: - branches: [ helm-deploy ] + branches: + - helm-deploy + - main workflow_dispatch: jobs: From c2ad499e2f6d0bcd5e1f33c86008e3481b8d0fe2 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 15:04:37 -0400 Subject: [PATCH 10/37] helm deployment and minikube test --- .github/workflows/helm-chart.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 38e52c7..2da49c2 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -25,8 +25,6 @@ jobs: context: ./opensampl/backend - name: grafana context: ./opensampl/server/grafana - #- name: migrations - # context: ./opensampl/migrations steps: - name: Checkout @@ -42,27 +40,25 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/ornl/opensampl-${{ matrix.service.name }} - tags: | - type=ref,event=pr - type=ref,event=branch - type=sha,prefix={{branch}}- - type=raw,value=latest,enable={{is_default_branch}} + - name: Set image tags + id: tags + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + PR_NUMBER=$(echo ${{ github.ref }} | awk -F'/' '{print $3}') + echo "tag=pr-${PR_NUMBER}" >> $GITHUB_OUTPUT + else + echo "tag=latest" >> $GITHUB_OUTPUT + fi - name: Build and push ${{ matrix.service.name }} image uses: docker/build-push-action@v5 with: context: ${{ matrix.service.context }} push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:${{ steps.tags.outputs.tag }} cache-from: type=gha,scope=${{ matrix.service.name }} cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} - + test-helm-chart: name: Test Helm Chart Deployment runs-on: ubuntu-latest From ba6a158a6cf49b30f99b7c329e03a79b7d23c3ca Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 15:39:48 -0400 Subject: [PATCH 11/37] Add missing DB init configmap and fixed image references --- helm/templates/backend/deployment.yaml | 2 +- helm/templates/db/db-init-configmap.yaml | 16 ++++++++++++++++ helm/templates/grafana/deployment.yaml | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 helm/templates/db/db-init-configmap.yaml diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index ef2dd75..efe2d88 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -32,7 +32,7 @@ spec: done; containers: - name: backend - image: {{ .Values.backend.image }} + image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} ports: - containerPort: {{ .Values.backend.port }} diff --git a/helm/templates/db/db-init-configmap.yaml b/helm/templates/db/db-init-configmap.yaml new file mode 100644 index 0000000..9dd7f5e --- /dev/null +++ b/helm/templates/db/db-init-configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "opensampl.fullname" . }}-db-init + labels: + {{- include "opensampl.labels" . | nindent 4 }} +data: + 020_create_grafana_user.sh: | + #!/bin/bash + set -e + + # This script creates additional database setup for Grafana + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + -- Grant necessary privileges + GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO ${POSTGRES_USER}; + EOSQL \ No newline at end of file diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index 5c3eca4..fa3a5a6 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: grafana - image: {{ .Values.grafana.image }} + image: "{{ .Values.grafana.image.repository }}:{{ .Values.grafana.image.tag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} ports: - containerPort: {{ .Values.grafana.port }} From 8ee95055d1a0d55d3528eb28cd66b929e8d1f127 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 15:58:28 -0400 Subject: [PATCH 12/37] Fix helm template naming format from .name to .fullname --- helm/templates/backend/deployment.yaml | 2 +- helm/templates/db/statefulset.yaml | 5 ++++- helm/templates/grafana/deployment.yaml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index efe2d88..a9f1708 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -38,7 +38,7 @@ spec: - containerPort: {{ .Values.backend.port }} env: - name: DATABASE_URL - value: "postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }}" + value: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} {{- with .Values.backend.env }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml index 27ab052..0ed53bf 100644 --- a/helm/templates/db/statefulset.yaml +++ b/helm/templates/db/statefulset.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "opensampl.labels" . | nindent 4 }} spec: - serviceName: {{ include "opensampl.name" . }}-db + serviceName: {{ include "opensampl.fullname" . }}-db replicas: 1 selector: matchLabels: @@ -25,6 +25,9 @@ spec: ports: - containerPort: {{ .Values.db.port }} name: postgres + env: + - name: PGDATA + value: /var/lib/postgres/data/pgdata envFrom: - secretRef: name: {{ include "opensampl.fullname" . }}-db-secret diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index fa3a5a6..82f0b3a 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -28,7 +28,7 @@ spec: - name: GF_DATABASE_TYPE value: postgres - name: GF_DATABASE_HOST - value: {{ include "opensampl.name" . }}-db:{{ .Values.db.port }} + value: {{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }} - name: GF_DATABASE_NAME value: {{ .Values.db.database }} - name: GF_DATABASE_USER From 1ebc47c3f72943477bb01d3588603502ccff1cd8 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 16:16:45 -0400 Subject: [PATCH 13/37] fix typos in db deployment --- helm/templates/db/statefulset.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml index 0ed53bf..5a33183 100644 --- a/helm/templates/db/statefulset.yaml +++ b/helm/templates/db/statefulset.yaml @@ -9,11 +9,15 @@ spec: replicas: 1 selector: matchLabels: - app: {{ include "opensampl.name" . }}-db + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: db template: metadata: labels: - app: {{ include "opensampl.name" . }}-db + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: db spec: securityContext: runAsUser: 999 @@ -27,7 +31,7 @@ spec: name: postgres env: - name: PGDATA - value: /var/lib/postgres/data/pgdata + value: /var/lib/postgresql/data/pgdata envFrom: - secretRef: name: {{ include "opensampl.fullname" . }}-db-secret From aa7b6fcc820dd1eb6c510bc9e3bd9b845961b418 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 17:04:05 -0400 Subject: [PATCH 14/37] fix volume write permisson for DB --- helm/templates/db/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml index 5a33183..1724be1 100644 --- a/helm/templates/db/statefulset.yaml +++ b/helm/templates/db/statefulset.yaml @@ -20,8 +20,8 @@ spec: role: db spec: securityContext: - runAsUser: 999 fsGroup: 999 + fsGroupChangePolicy: "OnRootMismatch" containers: - name: db image: "{{ .Values.db.image.repository }}:{{ .Values.db.image.tag }}" From 54447e92201a9facb9501012032378569de2efa5 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 17:32:00 -0400 Subject: [PATCH 15/37] diable migrations helm deploy --- .github/workflows/helm-chart.yml | 1 + helm/templates/migrations/migrations-job.yaml | 26 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 2da49c2..191faaf 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -83,6 +83,7 @@ jobs: --set db.image.tag=pr-${PR_NUMBER} \ --set backend.image.tag=pr-${PR_NUMBER} \ --set grafana.image.tag=pr-${PR_NUMBER} \ + --set migrations.enabled=false \ --wait \ --timeout 5m diff --git a/helm/templates/migrations/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml index 0fb36e8..3882065 100644 --- a/helm/templates/migrations/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -1,22 +1,20 @@ +{{- if .Values.migrations.enabled }} apiVersion: batch/v1 kind: Job metadata: name: {{ include "opensampl.fullname" . }}-migrations - labels: - {{- include "opensampl.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: template: - metadata: - labels: - app: {{ include "opensampl.name" . }} spec: - restartPolicy: OnFailure containers: - - name: migrations - image: {{ .Values.migrations.image }} - env: - - name: DATABASE_URL - value: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.name" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} \ No newline at end of file + - name: migrations + image: "{{ .Values.migrations.image.repository }}:{{ .Values.migrations.image.tag }}" + env: + - name: DATABASE_URL + value: postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} + envFrom: + - secretRef: + name: {{ include "opensampl.fullname" . }}-db-secret + - configMapRef: + name: {{ include "opensampl.fullname" . }}-db-config + restartPolicy: OnFailure \ No newline at end of file From 3c0f2fae012ef3bb72078c44b63cea9a4ca8eaed Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 17:36:13 -0400 Subject: [PATCH 16/37] deploy helm and test with minikube --- helm/templates/migrations/migrations-job.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helm/templates/migrations/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml index 3882065..8edd519 100644 --- a/helm/templates/migrations/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -17,4 +17,5 @@ spec: name: {{ include "opensampl.fullname" . }}-db-secret - configMapRef: name: {{ include "opensampl.fullname" . }}-db-config - restartPolicy: OnFailure \ No newline at end of file + restartPolicy: OnFailure +{{- end }} \ No newline at end of file From 49d745ba6dbfb56c8fa901b597cac6decf92e3a2 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 11:49:42 -0400 Subject: [PATCH 17/37] fixed ingress hosts --- helm/templates/ingress.yaml | 44 +++++++++++++------------- helm/values-prod.yaml | 63 +++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 helm/values-prod.yaml diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 40bdfa4..daf2d68 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -5,35 +5,37 @@ metadata: name: {{ include "opensampl.fullname" . }}-ingress labels: {{- include "opensampl.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} annotations: - kubernetes.io/ingress.class: {{ .Values.ingress.className }} - {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} tls: + {{- range .Values.ingress.tls }} - hosts: - - {{ .Values.ingress.host }} - secretName: {{ include "opensampl.fullname" . }}-tls + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} rules: - - host: {{ .Values.ingress.host }} + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} http: paths: - # Backend API - - path: /api - pathType: Prefix - backend: - service: - name: {{ include "opensampl.fullname" . }}-backend - port: - number: {{ .Values.backend.port }} - - # Grafana UI - - path: /grafana - pathType: Prefix + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} backend: service: - name: {{ include "opensampl.fullname" . }}-grafana + name: {{ include "opensampl.fullname" $ }}-{{ .service }} port: - number: {{ .Values.grafana.port }} -{{- end }} + number: {{ if eq .service "backend" }}{{ $.Values.backend.service.port }}{{ else }}{{ $.Values.grafana.port }}{{ end }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/values-prod.yaml b/helm/values-prod.yaml new file mode 100644 index 0000000..3b8aefb --- /dev/null +++ b/helm/values-prod.yaml @@ -0,0 +1,63 @@ +# values-production.yaml + +global: + imagePullPolicy: Always + +# Database configuration +db: + image: + repository: ghcr.io/ornl/opensampl-db + tag: latest + database: opensampl + username: opensampl + password: "admin" + port: 5432 + persistence: + size: 50Gi + storageClass: "longhorn" + +# Backend configuration +backend: + image: + repository: ghcr.io/ornl/opensampl-backend + tag: latest + replicas: 2 + service: + type: ClusterIP + port: 8000 + +# Grafana configuration +grafana: + image: + repository: ghcr.io/ornl/opensampl-grafana + tag: latest + adminUser: admin + adminPassword: "admin" + port: 3000 + service: + type: ClusterIP + +# Migrations (disabled until ready) +migrations: + enabled: false + +# Ingress configuration +ingress: + enabled: true + className: "nginx" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" # If using cert-manager + # nginx.ingress.kubernetes.io/ssl-redirect: "true" + hosts: + - host: opensampl-test.ornl.gov + paths: + - path: / + pathType: Prefix + service: backend + - path: /grafana + pathType: Prefix + service: grafana + tls: + - secretName: opensampl-tls + hosts: + - opensampl-test.ornl.gov \ No newline at end of file From b61cbdcde6bad04a8e19615174e7294999d3feef Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 11:56:18 -0400 Subject: [PATCH 18/37] update chart version --- helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 2abb772..4d154d5 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.0 +version: 0.1.1 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file From fce444defe59d026d3184f17822255bd344b1fd1 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 12:57:00 -0400 Subject: [PATCH 19/37] add grafana subpath to test UI in mariner --- "helm/Accept\357\200\272" | 0 helm/Chart.yaml | 2 +- helm/GET | 0 "helm/Host\357\200\272" | 0 "helm/User-Agent\357\200\272" | 0 helm/templates/grafana/deployment.yaml | 5 +++++ helm/values-prod.yaml | 2 ++ 7 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 "helm/Accept\357\200\272" create mode 100644 helm/GET create mode 100644 "helm/Host\357\200\272" create mode 100644 "helm/User-Agent\357\200\272" diff --git "a/helm/Accept\357\200\272" "b/helm/Accept\357\200\272" new file mode 100644 index 0000000..e69de29 diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 4d154d5..e01d14e 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.1 +version: 0.1.2 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/GET b/helm/GET new file mode 100644 index 0000000..e69de29 diff --git "a/helm/Host\357\200\272" "b/helm/Host\357\200\272" new file mode 100644 index 0000000..e69de29 diff --git "a/helm/User-Agent\357\200\272" "b/helm/User-Agent\357\200\272" new file mode 100644 index 0000000..e69de29 diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index 82f0b3a..d297ecd 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -38,6 +38,11 @@ spec: secretKeyRef: name: {{ include "opensampl.fullname" . }}-db-secret key: POSTGRES_PASSWORD + # For testing in Mariner + - name: GF_SERVER_ROOT_URL + value: {{ .Values.grafana.rootUrl | default "http://localhost:3000" }} + - name: GF_SERVER_SERVE_FROM_SUB_PATH + value: {{ .Values.grafana.serveFromSubPath | default "false" | quote }} readinessProbe: httpGet: path: /login diff --git a/helm/values-prod.yaml b/helm/values-prod.yaml index 3b8aefb..b7df5f2 100644 --- a/helm/values-prod.yaml +++ b/helm/values-prod.yaml @@ -36,6 +36,8 @@ grafana: port: 3000 service: type: ClusterIP + rootUrl: "https://opensampl-test.ornl.gov/grafana" + serveFromSubPath: true # Migrations (disabled until ready) migrations: From a015828a192d455d1fe3719ac12644c86177fa7d Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 13:19:28 -0400 Subject: [PATCH 20/37] remove subpath for grafana so it hosts at opensampl-test.ornl.gov --- helm/Chart.yaml | 2 +- helm/{values-prod.yaml => values-mariner.yaml} | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename helm/{values-prod.yaml => values-mariner.yaml} (92%) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index e01d14e..a0c7f3e 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.2 +version: 0.1.3 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/values-prod.yaml b/helm/values-mariner.yaml similarity index 92% rename from helm/values-prod.yaml rename to helm/values-mariner.yaml index b7df5f2..14825bb 100644 --- a/helm/values-prod.yaml +++ b/helm/values-mariner.yaml @@ -36,8 +36,8 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/grafana" - serveFromSubPath: true + rootUrl: "https://opensampl-test.ornl.gov/" + serveFromSubPath: false # Migrations (disabled until ready) migrations: @@ -53,10 +53,10 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: / + - path: /api pathType: Prefix service: backend - - path: /grafana + - path: / pathType: Prefix service: grafana tls: From da4dd431ef6b790f8a3811d94d116c229d6efe7a Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 13:53:48 -0400 Subject: [PATCH 21/37] try and host grafana at opensampl.ornl.gov --- helm/Chart.yaml | 2 +- helm/values-mariner.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index a0c7f3e..2aa74f8 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.3 +version: 0.1.4 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 14825bb..f489fc9 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -53,7 +53,7 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /api + - path: /backend pathType: Prefix service: backend - path: / From 450c3ad55ae450192c282f8fd006a82a916c235b Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 15:56:56 -0400 Subject: [PATCH 22/37] move grafana ui to /grafana to avoid API collisions --- helm/Chart.yaml | 2 +- helm/templates/grafana/deployment.yaml | 4 +-- helm/templates/ingress.yaml | 43 +++++++++-------------- helm/values-mariner.yaml | 8 ++--- opensampl/backend/{README.d => README.md} | 0 5 files changed, 23 insertions(+), 34 deletions(-) rename opensampl/backend/{README.d => README.md} (100%) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 2aa74f8..92e779e 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.4 +version: 0.1.5 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index d297ecd..b06cf61 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -40,9 +40,9 @@ spec: key: POSTGRES_PASSWORD # For testing in Mariner - name: GF_SERVER_ROOT_URL - value: {{ .Values.grafana.rootUrl | default "http://localhost:3000" }} + value: "https://opensampl-test.ornl.gov/grafana/" - name: GF_SERVER_SERVE_FROM_SUB_PATH - value: {{ .Values.grafana.serveFromSubPath | default "false" | quote }} + value: "true" readinessProbe: httpGet: path: /login diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index daf2d68..628007a 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -2,40 +2,29 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "opensampl.fullname" . }}-ingress - labels: - {{- include "opensampl.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} + name: {{ include "opensampl.fullname" . }} annotations: - {{- toYaml . | nindent 4 }} - {{- end }} + {{- toYaml .Values.ingress.annotations | nindent 4 }} spec: - {{- if .Values.ingress.className }} ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + {{- toYaml .Values.ingress.tls | nindent 4 }} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ (index .Values.ingress.hosts 0).host }} http: paths: - {{- range .paths }} - - path: {{ .path }} - pathType: {{ .pathType }} + - path: /(.*) + pathType: Prefix backend: service: - name: {{ include "opensampl.fullname" $ }}-{{ .service }} + name: {{ include "opensampl.fullname" . }}-backend port: - number: {{ if eq .service "backend" }}{{ $.Values.backend.service.port }}{{ else }}{{ $.Values.grafana.port }}{{ end }} - {{- end }} - {{- end }} -{{- end }} \ No newline at end of file + number: 80 + - path: /grafana(/|$)(.*) + pathType: Prefix + backend: + service: + name: {{ include "opensampl.fullname" . }}-grafana + port: + number: {{ .Values.grafana.port }} +{{- end }} diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index f489fc9..7eee203 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -36,8 +36,8 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/" - serveFromSubPath: false + rootUrl: "https://opensampl-test.ornl.gov/grafana" + serveFromSubPath: true # Migrations (disabled until ready) migrations: @@ -53,10 +53,10 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /backend + - path: /(.*) pathType: Prefix service: backend - - path: / + - path: /grafana(/|$)(.*) pathType: Prefix service: grafana tls: diff --git a/opensampl/backend/README.d b/opensampl/backend/README.md similarity index 100% rename from opensampl/backend/README.d rename to opensampl/backend/README.md From 1d2d06f21c33af4de4a86b353e24ae1624765ab9 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 16:24:07 -0400 Subject: [PATCH 23/37] fix path to servce /api/docs --- helm/Chart.yaml | 2 +- helm/templates/backend/service.yaml | 4 ++-- helm/templates/ingress.yaml | 13 +++++++++++-- helm/values-mariner.yaml | 6 +++--- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 92e779e..3c0113f 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.5 +version: 0.1.6 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/templates/backend/service.yaml b/helm/templates/backend/service.yaml index 2f091c5..c38c02c 100644 --- a/helm/templates/backend/service.yaml +++ b/helm/templates/backend/service.yaml @@ -10,5 +10,5 @@ spec: app: {{ include "opensampl.name" . }}-backend ports: - name: http - port: {{ .Values.backend.port }} - targetPort: {{ .Values.backend.port }} + port: 80 + targetPort: 8000 \ No newline at end of file diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 628007a..bb17380 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -13,13 +13,14 @@ spec: - host: {{ (index .Values.ingress.hosts 0).host }} http: paths: - - path: /(.*) + - path: /api(/|$)(.*) pathType: Prefix backend: service: name: {{ include "opensampl.fullname" . }}-backend port: - number: 80 + number: {{ .Values.backend.service.port }} + - path: /grafana(/|$)(.*) pathType: Prefix backend: @@ -27,4 +28,12 @@ spec: name: {{ include "opensampl.fullname" . }}-grafana port: number: {{ .Values.grafana.port }} + + - path: / + pathType: Prefix + backend: + service: + name: {{ include "opensampl.fullname" . }}-grafana + port: + number: {{ .Values.grafana.port }} {{- end }} diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 7eee203..09966dc 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -21,7 +21,7 @@ backend: image: repository: ghcr.io/ornl/opensampl-backend tag: latest - replicas: 2 + replicas: 1 service: type: ClusterIP port: 8000 @@ -36,7 +36,7 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/grafana" + rootUrl: "https://opensampl-test.ornl.gov/grafana/" serveFromSubPath: true # Migrations (disabled until ready) @@ -53,7 +53,7 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /(.*) + - path: /api(/|$)(.*) pathType: Prefix service: backend - path: /grafana(/|$)(.*) From c1bc729b25118831937cb577b343d75c52157d0c Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 16:36:56 -0400 Subject: [PATCH 24/37] fix backend ingress section to point to backend --- helm/templates/ingress.yaml | 4 ++-- helm/values-mariner.yaml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index bb17380..957857b 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -33,7 +33,7 @@ spec: pathType: Prefix backend: service: - name: {{ include "opensampl.fullname" . }}-grafana + name: {{ include "opensampl.fullname" . }}-backend port: - number: {{ .Values.grafana.port }} + number: {{ .Values.backend.service.port }} {{- end }} diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 09966dc..a222296 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -56,6 +56,9 @@ ingress: - path: /api(/|$)(.*) pathType: Prefix service: backend + - path: /docs(/|$)(.*) + pathType: Prefix + service: backend - path: /grafana(/|$)(.*) pathType: Prefix service: grafana From 0604f8d8bb21b83bd9eed198c6d9989dac074155 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 17:14:01 -0400 Subject: [PATCH 25/37] fix backend 503 error --- helm/Chart.yaml | 2 +- helm/templates/backend/deployment.yaml | 17 ++++++++++++++--- helm/templates/backend/service.yaml | 6 ++++-- helm/values-mariner.yaml | 23 +++++++++++++---------- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 3c0113f..2d20e33 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.6 +version: 0.1.7 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index a9f1708..16d48c1 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -35,10 +35,21 @@ spec: image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} ports: - - containerPort: {{ .Values.backend.port }} + - containerPort: 8000 env: - name: DATABASE_URL - value: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} + value: postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} + - name: POSTGRES_USER + value: {{ .Values.db.username }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "opensampl.fullname" . }}-db-secret + key: POSTGRES_PASSWORD + - name: BACKEND_LOG_LEVEL + value: {{ .Values.backend.logLevel | default "INFO" | quote }} + - name: USE_API_KEY + value: {{ .Values.backend.useApiKey | default "false" | quote }} {{- with .Values.backend.env }} {{- toYaml . | nindent 12 }} - {{- end }} + {{- end }} \ No newline at end of file diff --git a/helm/templates/backend/service.yaml b/helm/templates/backend/service.yaml index c38c02c..d2be513 100644 --- a/helm/templates/backend/service.yaml +++ b/helm/templates/backend/service.yaml @@ -7,8 +7,10 @@ metadata: spec: type: ClusterIP selector: - app: {{ include "opensampl.name" . }}-backend + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: backend ports: - name: http - port: 80 + port: {{ .Values.backend.service.port }} targetPort: 8000 \ No newline at end of file diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index a222296..e014271 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -25,6 +25,9 @@ backend: service: type: ClusterIP port: 8000 + logLevel: "INFO" + useApiKey: false + env: [] # Grafana configuration grafana: @@ -36,7 +39,7 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/grafana/" + rootUrl: "https://opensampl-test.ornl.gov/grafana" serveFromSubPath: true # Migrations (disabled until ready) @@ -53,15 +56,15 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /api(/|$)(.*) - pathType: Prefix - service: backend - - path: /docs(/|$)(.*) - pathType: Prefix - service: backend - - path: /grafana(/|$)(.*) - pathType: Prefix - service: grafana + - path: /api + pathType: prefix + service: backend + - path: /docs + pathType: Prefix + service: backend + - path: /grafana(/|$)(.*) + pathType: Prefix + service: grafana tls: - secretName: opensampl-tls hosts: From eadb2f7c71c1e6ff58051214bc46878ff8257e11 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 17:26:32 -0400 Subject: [PATCH 26/37] fix indentation on service.port --- helm/values-mariner.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index e014271..54473bf 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -25,9 +25,9 @@ backend: service: type: ClusterIP port: 8000 - logLevel: "INFO" - useApiKey: false - env: [] + logLevel: "INFO" + useApiKey: false + env: [] # Grafana configuration grafana: From 7d94b670439737393d08e4862d645a55c02b9b28 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 17:34:41 -0400 Subject: [PATCH 27/37] deploy backend to /api and /docs --- helm/values.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index dbae2f6..753c5cb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -24,13 +24,12 @@ backend: repository: ghcr.io/ornl/opensampl-backend tag: latest replicas: 1 - port: 8000 - env: - # Additional backend environment variables - - name: LOG_LEVEL - value: INFO - - name: USE_API_KEY - value: "false" + service: + type: ClusterIP + port: 8000 + logLevel: "INFO" + useApiKey: false + env: [] # Grafana service grafana: From 847f2bc62fb3f6c0fc41bba70ea01fe226e2d30f Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 18:33:25 -0400 Subject: [PATCH 28/37] move grafana to the top of hosts --- helm/templates/grafana/deployment.yaml | 2 +- helm/values-mariner.yaml | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index b06cf61..9f132b2 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -40,7 +40,7 @@ spec: key: POSTGRES_PASSWORD # For testing in Mariner - name: GF_SERVER_ROOT_URL - value: "https://opensampl-test.ornl.gov/grafana/" + value: "https://opensampl-test.ornl.gov/grafana" - name: GF_SERVER_SERVE_FROM_SUB_PATH value: "true" readinessProbe: diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 54473bf..9c72706 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -56,15 +56,15 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /api - pathType: prefix - service: backend - - path: /docs - pathType: Prefix - service: backend - - path: /grafana(/|$)(.*) - pathType: Prefix - service: grafana + - path: /grafana + pathType: Prefix + service: grafana + - path: /api + pathType: prefix + service: backend + - path: /docs + pathType: Prefix + service: backend tls: - secretName: opensampl-tls hosts: From 19bec243c7420fe3834e9c1b6d21a92471bf9cee Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 18:39:10 -0400 Subject: [PATCH 29/37] add subpath value for grafana --- helm/templates/grafana/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index 9f132b2..cf27599 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -42,7 +42,7 @@ spec: - name: GF_SERVER_ROOT_URL value: "https://opensampl-test.ornl.gov/grafana" - name: GF_SERVER_SERVE_FROM_SUB_PATH - value: "true" + value: {{ .Values.grafana.serveFromSubPath | default "false" | quote }} readinessProbe: httpGet: path: /login From abf5b07676a0ec62ce702ba294997edb74b7d9e3 Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 3 Nov 2025 11:17:17 -0500 Subject: [PATCH 30/37] fix /grafana path --- helm/templates/ingress.yaml | 52 +++++++++++++++++++------------------ helm/values-mariner.yaml | 3 --- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 957857b..daf2d68 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -2,38 +2,40 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "opensampl.fullname" . }} + name: {{ include "opensampl.fullname" . }}-ingress + labels: + {{- include "opensampl.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} annotations: - {{- toYaml .Values.ingress.annotations | nindent 4 }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: + {{- if .Values.ingress.className }} ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} tls: - {{- toYaml .Values.ingress.tls | nindent 4 }} + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} rules: - - host: {{ (index .Values.ingress.hosts 0).host }} + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} http: paths: - - path: /api(/|$)(.*) - pathType: Prefix + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} backend: service: - name: {{ include "opensampl.fullname" . }}-backend + name: {{ include "opensampl.fullname" $ }}-{{ .service }} port: - number: {{ .Values.backend.service.port }} - - - path: /grafana(/|$)(.*) - pathType: Prefix - backend: - service: - name: {{ include "opensampl.fullname" . }}-grafana - port: - number: {{ .Values.grafana.port }} - - - path: / - pathType: Prefix - backend: - service: - name: {{ include "opensampl.fullname" . }}-backend - port: - number: {{ .Values.backend.service.port }} -{{- end }} + number: {{ if eq .service "backend" }}{{ $.Values.backend.service.port }}{{ else }}{{ $.Values.grafana.port }}{{ end }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 9c72706..0d96384 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -59,9 +59,6 @@ ingress: - path: /grafana pathType: Prefix service: grafana - - path: /api - pathType: prefix - service: backend - path: /docs pathType: Prefix service: backend From f97fe259028382eed3f2b9ee732a0fddb667250f Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 3 Nov 2025 11:50:21 -0500 Subject: [PATCH 31/37] remove /docs and server root --- helm/values-mariner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 0d96384..915bf7e 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -59,7 +59,7 @@ ingress: - path: /grafana pathType: Prefix service: grafana - - path: /docs + - path: / pathType: Prefix service: backend tls: From 8ab54807b81c3ec63c0016e9b1db1eaae6c59c87 Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 14:22:57 -0400 Subject: [PATCH 32/37] Clean up helm chart files --- "helm/Accept\357\200\272" | 0 helm/GET | 0 "helm/Host\357\200\272" | 0 "helm/User-Agent\357\200\272" | 0 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 "helm/Accept\357\200\272" delete mode 100644 helm/GET delete mode 100644 "helm/Host\357\200\272" delete mode 100644 "helm/User-Agent\357\200\272" diff --git "a/helm/Accept\357\200\272" "b/helm/Accept\357\200\272" deleted file mode 100644 index e69de29..0000000 diff --git a/helm/GET b/helm/GET deleted file mode 100644 index e69de29..0000000 diff --git "a/helm/Host\357\200\272" "b/helm/Host\357\200\272" deleted file mode 100644 index e69de29..0000000 diff --git "a/helm/User-Agent\357\200\272" "b/helm/User-Agent\357\200\272" deleted file mode 100644 index e69de29..0000000 From 829222355c9dc6f2ddfdd194426aef75798f70cb Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 15:33:29 -0400 Subject: [PATCH 33/37] update cluster specific dns opensampl-test.eig3-orc1.ornl.gov --- ...{values-mariner.yaml => values-cluster-overrides.yaml} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename helm/{values-mariner.yaml => values-cluster-overrides.yaml} (86%) diff --git a/helm/values-mariner.yaml b/helm/values-cluster-overrides.yaml similarity index 86% rename from helm/values-mariner.yaml rename to helm/values-cluster-overrides.yaml index 915bf7e..f630251 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-cluster-overrides.yaml @@ -1,4 +1,4 @@ -# values-production.yaml +# Kubernetes cluster specific overrides. global: imagePullPolicy: Always @@ -39,7 +39,7 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/grafana" + rootUrl:"https://opensampl-test.eig3-orc1.ornl.gov/grafana" serveFromSubPath: true # Migrations (disabled until ready) @@ -54,7 +54,7 @@ ingress: cert-manager.io/cluster-issuer: "letsencrypt-prod" # If using cert-manager # nginx.ingress.kubernetes.io/ssl-redirect: "true" hosts: - - host: opensampl-test.ornl.gov + - host: opensampl-test.eig3-orc1.ornl.gov paths: - path: /grafana pathType: Prefix @@ -65,4 +65,4 @@ ingress: tls: - secretName: opensampl-tls hosts: - - opensampl-test.ornl.gov \ No newline at end of file + - opensampl-test.eig3-orc1.ornl.gov \ No newline at end of file From fe6907f9b6b3e399fca28dc57048ddc293d7f3fb Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 15:39:31 -0400 Subject: [PATCH 34/37] build chart artifiact v0.1.8 --- helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 2d20e33..b0e5438 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.7 +version: 0.1.8 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file From 114d050c43466f663c868d86f81340c38679c029 Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 16:08:21 -0400 Subject: [PATCH 35/37] add values-override and helm lint values-override to the helm workflow --- .github/workflows/helm-chart.yml | 13 ++++++++++++- helm/Chart.yaml | 2 +- helm/values-cluster-overrides.yaml | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 191faaf..1d7c638 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -80,10 +80,12 @@ jobs: PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}') helm install test-opensampl . \ + -f values-cluster-overrides.yaml \ --set db.image.tag=pr-${PR_NUMBER} \ --set backend.image.tag=pr-${PR_NUMBER} \ --set grafana.image.tag=pr-${PR_NUMBER} \ --set migrations.enabled=false \ + --set ingress.enabled=false \ --wait \ --timeout 5m @@ -121,8 +123,17 @@ jobs: version: v3.14.0 - name: Helm Lint - run: helm lint ./helm + run: | + helm lint ./helm + helm lint ./helm -f ./helm/values-cluster-overrides.yaml + - name: Render Helm Chart + run: | + helm template opensampl ./helm \ + -f ./helm/values-cluster-overrides.yaml \ + --namespace opensampl-test \ + > /tmp/opensampl-rendered.yaml + - name: Package Chart run: helm package ./helm diff --git a/helm/Chart.yaml b/helm/Chart.yaml index b0e5438..b471011 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.8 +version: 0.1.9 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/values-cluster-overrides.yaml b/helm/values-cluster-overrides.yaml index f630251..5d36c43 100644 --- a/helm/values-cluster-overrides.yaml +++ b/helm/values-cluster-overrides.yaml @@ -39,7 +39,7 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl:"https://opensampl-test.eig3-orc1.ornl.gov/grafana" + rootUrl: "https://opensampl-test.eig3-orc1.ornl.gov/grafana" serveFromSubPath: true # Migrations (disabled until ready) From 0c9a7151030b008e2573621b3419abf818783cd0 Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 16:22:05 -0400 Subject: [PATCH 36/37] push chart 0.2.0 --- .github/workflows/helm-chart.old | 76 -------------------------------- .github/workflows/helm-chart.yml | 1 - helm/Chart.yaml | 2 +- 3 files changed, 1 insertion(+), 78 deletions(-) delete mode 100644 .github/workflows/helm-chart.old diff --git a/.github/workflows/helm-chart.old b/.github/workflows/helm-chart.old deleted file mode 100644 index 6e234c0..0000000 --- a/.github/workflows/helm-chart.old +++ /dev/null @@ -1,76 +0,0 @@ -name: Helm Chart CI/CD - -on: - push: - branches: [ helm-deploy ] - pull_request: - branches: [ helm-deploy ] - workflow_dispatch: - -jobs: - build-docker-images: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - strategy: - matrix: - service: - - name: db - context: ./opensampl/db - - name: backend - context: ./opensampl/backend - - name: grafana - context: ./opensampl/server/grafana - #- name: migrations - # context: ./opensampl/migrations - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push ${{ matrix.service.name }} image - uses: docker/build-push-action@v5 - with: - context: ${{ matrix.service.context }} - push: true - tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest - cache-from: type=gha,scope=${{ matrix.service.name }} - cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} - - helm: - runs-on: ubuntu-latest - needs: build-docker-images - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Helm - uses: azure/setup-helm@v3 - with: - version: v3.14.0 - - - name: Helm Lint - run: helm lint ./helm - - - name: Package Chart - run: helm package ./helm - - - name: Push to GHCR - env: - CR_PAT: ${{ secrets.GITHUB_TOKEN }} - run: | - helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts - diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 1d7c638..0e9112b 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -80,7 +80,6 @@ jobs: PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}') helm install test-opensampl . \ - -f values-cluster-overrides.yaml \ --set db.image.tag=pr-${PR_NUMBER} \ --set backend.image.tag=pr-${PR_NUMBER} \ --set grafana.image.tag=pr-${PR_NUMBER} \ diff --git a/helm/Chart.yaml b/helm/Chart.yaml index b471011..f377aed 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.9 +version: 0.2.0 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file From e2c1c7db2c0e8de8b21a82161a2d43c7a1adb77b Mon Sep 17 00:00:00 2001 From: 7va Date: Tue, 18 Aug 2026 08:57:21 -0400 Subject: [PATCH 37/37] pin busy box image to eliminate ornl pull limits --- helm/Chart.yaml | 2 +- helm/templates/backend/deployment.yaml | 2 +- helm/values-cluster-overrides.yaml | 2 +- helm/values.yaml | 5 +++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index f377aed..77decd3 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.2.0 +version: 0.2.1 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index 16d48c1..3976f1d 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -21,7 +21,7 @@ spec: spec: initContainers: - name: wait-for-db - image: busybox + image: "{{ .Values.busybox.image.repository }}:{{ .Values.busybox.image.tag }}" command: - sh - -c diff --git a/helm/values-cluster-overrides.yaml b/helm/values-cluster-overrides.yaml index 5d36c43..b27719c 100644 --- a/helm/values-cluster-overrides.yaml +++ b/helm/values-cluster-overrides.yaml @@ -1,4 +1,4 @@ -# Kubernetes cluster specific overrides. +# Laboratory and Kubernetes cluster specific overrides. global: imagePullPolicy: Always diff --git a/helm/values.yaml b/helm/values.yaml index 753c5cb..429cfdf 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -2,6 +2,11 @@ global: imagePullPolicy: IfNotPresent +busybox: + image: + repository: ghcr.io/ornl/opensampl-busybox + tag: "1.36.1" + # PostgreSQL Database db: image: