feat(stack): add gizmosql and redpanda-connect

artifacts/docs/README.md

@@ -16,7 +16,7 @@ Services: adminer, cloudflared, portainer, portainer-agent, postgraphile, postgr
 
 ### event-streaming
 
-Services: debezium, debezium-postgres-connector, postgres, redpanda, redpanda-console, sqitch, traefik
+Services: debezium, debezium-postgres-connector, gizmosql, postgres, redpanda, redpanda-connect, redpanda-console, sqitch, traefik
 
 ### recommendation
 
@@ -73,6 +73,10 @@ You cannot access the search engine via a web interface.
 
 You cannot access the ip geolocator via a web interface.
 
+### gizmosql
+
+You can query the event stream's parquet lakehouse using any Arrow Flight SQL client at gizmosql:31337.
+
 ### grafana
 
 You can access the observation dashboard at [grafana.app.localhost](https://grafana.app.localhost/).
@@ -142,6 +146,10 @@ You cannot access the caching system via a web interface.
 
 You can access the event streaming platform's ui as described under `redpanda-console`.
 
+### redpanda-connect
+
+You can track the event stream ingestion using the pipeline's readiness endpoint.
+
 ### redpanda-console
 
 You can access the event streaming platform's ui at [redpanda.app.localhost](https://redpanda.app.localhost/).

src/development/gizmosql/compose.yaml

@@ -0,0 +1,58 @@
+secrets:
+  gizmosql-password:
+    # The gizmosql server's password.
+    file: ~~/artifacts/secrets/gizmosql-password.secret
+  r2-account-id:
+    # The event stream analytics' cloud storage account identifier.
+    file: ~~/artifacts/secrets/r2-account-id.secret
+  r2-aws-credentials-access-key:
+    # The event stream analytics' cloud storage secret access key.
+    file: ~~/artifacts/secrets/r2-aws-credentials-access-key.secret
+  r2-aws-credentials-access-key-id:
+    # The event stream analytics' cloud storage access key identifier.
+    file: ~~/artifacts/secrets/r2-aws-credentials-access-key-id.secret
+services:
+  gizmosql:
+    # You can query the event stream's parquet lakehouse using any Arrow Flight SQL client.
+    command: /run/entrypoint.sh
+    deploy:
+      labels:
+        - dargstack.profiles=event-streaming
+    entrypoint: sh
+    environment:
+      GIZMOSQL_USERNAME: gizmosql
+      INIT_SQL_COMMANDS_FILE: /tmp/gizmosql-init.sql
+      PRINT_QUERIES: "1"
+      TLS_ENABLED: "0"
+    healthcheck:
+      test: ["CMD-SHELL", "GIZMOSQL_PASSWORD=$$(cat /run/environment-variables/GIZMOSQL_PASSWORD) gizmosql_client --host 127.0.0.1 --port 31337 --username gizmosql --command 'SELECT 1'"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s
+    image: gizmodata/gizmosql:v1.35.1
+    ports: # dargstack:dev-only
+      - 31337:31337 # dargstack:dev-only
+    secrets:
+      - source: gizmosql-password
+        target: /run/environment-variables/GIZMOSQL_PASSWORD
+      - source: r2-account-id
+        target: /run/environment-variables/R2_ACCOUNT_ID
+      - source: r2-aws-credentials-access-key-id
+        target: /run/environment-variables/R2_ACCESS_KEY_ID
+      - source: r2-aws-credentials-access-key
+        target: /run/environment-variables/R2_SECRET_ACCESS_KEY
+    volumes:
+      - ./configurations/entrypoint.sh:/run/entrypoint.sh:ro
+      - ./configurations/init.sql.template:/run/init.sql.template:ro
+x-dargstack:
+  secrets:
+    gizmosql-password:
+      special_characters: false
+      type: random_string
+    r2-account-id:
+      type: third_party
+    r2-aws-credentials-access-key:
+      type: third_party
+    r2-aws-credentials-access-key-id:
+      type: third_party

src/development/gizmosql/configurations/entrypoint.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+set -eu
+
+# START of maevsi entrypoint script customization
+ENVIRONMENT_VARIABLES_PATH="/run/environment-variables"
+
+is_valid_var_name() {
+  case "$1" in
+    ''|[!a-zA-Z_]*|*[!a-zA-Z0-9_]*) return 1 ;;
+    *) return 0 ;;
+  esac
+}
+
+load_env_file() {
+  file="$1"
+  name=$(basename "$file")
+  is_valid_var_name "$name" || return 0
+  value=$(cat "$file")
+  export "$name=$value"
+}
+
+load_environment_variables() {
+  [ -d "$ENVIRONMENT_VARIABLES_PATH" ] || return 0
+  set -- "$ENVIRONMENT_VARIABLES_PATH"/*
+  [ -e "$1" ] || return 0
+
+  for file in "$ENVIRONMENT_VARIABLES_PATH"/*; do
+    [ -f "$file" ] && load_env_file "$file"
+  done
+}
+
+load_environment_variables
+# END of maevsi entrypoint script customization
+
+# DuckDB in this image has no getenv() function, so the R2 values are rendered
+# into init.sql. sed treats `&` and `\` specially in replacements, so escape
+# them first (R2 keys are hex today, but this keeps arbitrary values safe).
+escape_sed_replacement() {
+  printf '%s\n' "$1" | sed 's/[&\\]/\\&/g'
+}
+
+R2_ACCOUNT_ID="$(escape_sed_replacement "${R2_ACCOUNT_ID}")"
+R2_ACCESS_KEY_ID="$(escape_sed_replacement "${R2_ACCESS_KEY_ID}")"
+R2_SECRET_ACCESS_KEY="$(escape_sed_replacement "${R2_SECRET_ACCESS_KEY}")"
+
+sed -e "s|__R2_ACCOUNT_ID__|${R2_ACCOUNT_ID}|g" \
+    -e "s|__R2_ACCESS_KEY_ID__|${R2_ACCESS_KEY_ID}|g" \
+    -e "s|__R2_SECRET_ACCESS_KEY__|${R2_SECRET_ACCESS_KEY}|g" \
+    /run/init.sql.template > /tmp/gizmosql-init.sql
+
+exec /opt/gizmosql/scripts/start_gizmosql.sh

src/development/gizmosql/configurations/init.sql.template

@@ -0,0 +1,38 @@
+INSTALL httpfs;
+LOAD httpfs;
+
+CREATE SECRET r2_secret (TYPE r2, KEY_ID '__R2_ACCESS_KEY_ID__', SECRET '__R2_SECRET_ACCESS_KEY__', ACCOUNT_ID '__R2_ACCOUNT_ID__');
+
+CREATE SCHEMA IF NOT EXISTS analytics;
+
+-- The event stream's parquet lakehouse, written by redpanda-connect to R2
+-- (bucket "test", prefix "events/"). The raw parquet columns are strings, so
+-- ids and timestamps are cast to proper types (try_cast keeps bad values NULL).
+CREATE OR REPLACE VIEW analytics.events AS
+SELECT
+  try_cast(id AS UUID) AS id,
+  name,
+  slug,
+  try_cast(start AS TIMESTAMPTZ) AS start,
+  try_cast("end" AS TIMESTAMPTZ) AS "end",
+  visibility,
+  is_archived,
+  is_in_person,
+  is_remote,
+  language,
+  url,
+  try_cast(address_id AS UUID) AS address_id,
+  guest_count_maximum,
+  try_cast(created_at AS TIMESTAMPTZ) AS created_at,
+  try_cast(created_by AS UUID) AS created_by,
+  operation
+FROM read_parquet('r2://test/events/*.parquet');
+
+-- Example aggregate over the lakehouse.
+CREATE OR REPLACE VIEW analytics.events_daily AS
+SELECT
+  try_cast(created_at AS DATE) AS day,
+  count(*) AS events
+FROM read_parquet('r2://test/events/*.parquet')
+GROUP BY 1
+ORDER BY 1;

src/development/redpanda-connect/compose.yaml

@@ -0,0 +1,30 @@
+secrets:
+  r2-aws-credentials:
+    # The event stream ingestion's s3 credentials file.
+    file: ~~/artifacts/secrets/r2-aws-credentials.secret
+services:
+  redpanda-connect:
+    # You can track the event stream's parquet ingestion using the pipeline's readiness endpoint.
+    deploy:
+      labels:
+        - dargstack.profiles=event-streaming
+    environment:
+      AWS_SHARED_CREDENTIALS_FILE: /run/secrets/r2-aws-credentials
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider http://127.0.0.1:4195/ready"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    image: redpandadata/connect:4.103.1
+    secrets:
+      - r2-aws-credentials
+    volumes:
+      - ./configurations/config.yaml:/connect.yaml:ro
+x-dargstack:
+  secrets:
+    r2-aws-credentials:
+      template: |
+        [default]
+        aws_access_key_id = {{secret:r2-aws-credentials-access-key-id}}
+        aws_secret_access_key = {{secret:r2-aws-credentials-access-key}}

src/development/redpanda-connect/configurations/config.yaml

@@ -0,0 +1,77 @@
+input:
+  kafka_franz:
+    seed_brokers: ["redpanda:9092"]
+    topics: ["vibetype.vibetype.event"]
+    consumer_group: "rp_connect_r2_ingest"
+pipeline:
+  processors:
+    # Tombstone messages (Debezium deletes) carry no payload and must be dropped.
+    - bloblang: |
+        root = if this.payload.after.or(this.after) == null { deleted() }
+    - mapping: |
+        root = this.payload.after.or(this.after)
+        root.operation = this.payload.op.or(this.op)
+output:
+  aws_s3:
+    bucket: test
+    endpoint: https://36b2a924f7d2470843ca1728530531cf.r2.cloudflarestorage.com
+    region: weur
+    force_path_style_urls: true
+    credentials:
+      profile: default
+    path: 'events/${! timestamp_unix_nano() }-${! uuid_v4() }.parquet'
+    batching:
+      count: 10000
+      period: 15s
+      processors:
+        - parquet_encode:
+            schema:
+              - name: id
+                type: UTF8
+                optional: true
+              - name: name
+                type: UTF8
+                optional: true
+              - name: slug
+                type: UTF8
+                optional: true
+              - name: start
+                type: UTF8
+                optional: true
+              - name: end
+                type: UTF8
+                optional: true
+              - name: visibility
+                type: UTF8
+                optional: true
+              - name: is_archived
+                type: BOOLEAN
+                optional: true
+              - name: is_in_person
+                type: BOOLEAN
+                optional: true
+              - name: is_remote
+                type: BOOLEAN
+                optional: true
+              - name: language
+                type: UTF8
+                optional: true
+              - name: url
+                type: UTF8
+                optional: true
+              - name: address_id
+                type: UTF8
+                optional: true
+              - name: guest_count_maximum
+                type: INT64
+                optional: true
+              - name: created_at
+                type: UTF8
+                optional: true
+              - name: created_by
+                type: UTF8
+                optional: true
+              - name: operation
+                type: UTF8
+                optional: true
+            default_compression: zstd

src/production/gizmosql/compose.yaml

@@ -0,0 +1,5 @@
+services:
+  gizmosql:
+    deploy:
+      update_config:
+        order: stop-first

src/production/redpanda-connect/compose.yaml

@@ -0,0 +1,5 @@
+services:
+  redpanda-connect:
+    deploy:
+      update_config:
+        order: stop-first