Add Baseshift extension

.github/workflows/baseshift.yml

@@ -0,0 +1,111 @@
+name: LocalStack Baseshift Extension Tests
+
+on:
+  schedule:
+    - cron: '0 2 * * 1-5'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/workflows/baseshift.yml
+      - 'baseshift/**'
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/baseshift.yml
+      - 'baseshift/**'
+  workflow_dispatch:
+
+env:
+  LOCALSTACK_DISABLE_EVENTS: "1"
+  LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+
+jobs:
+  integration-tests:
+    name: Run Baseshift Extension Tests (${{ matrix.emulator }}, ${{ matrix.tag }})
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        # the Snowflake emulator image also provides the AWS services, and is used for the end-to-end demo
+        emulator:
+          - aws
+          - snowflake
+        tag:
+          - latest
+          - dev
+    env:
+      LOCALSTACK_EMULATOR: ${{ matrix.emulator }}
+      LOCALSTACK_TAG: ${{ matrix.tag }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Install lstk CLI
+        # version is pinned; there is no lockfile for global CLI tools
+        run: | # zizmor: ignore[adhoc-packages]
+          npm install -g @localstack/lstk@1.2.0
+          lstk --version
+
+      - name: Build extension
+        run: |
+          cd baseshift
+          make install
+          make lint
+          make dist
+
+      - name: Start LocalStack with extension
+        run: |
+          cd baseshift
+
+          # Baseshift clone images are private to each customer, hence we use a plain
+          # Postgres image as a stand-in (clones also accept connections without a password)
+          docker pull postgres:17
+
+          # mount the built extension into the container, so LocalStack can install it at startup
+          CONFIG_FILE="$(lstk config path)"
+          mkdir -p "$(dirname "$CONFIG_FILE")"
+          cat > "$CONFIG_FILE" <<EOT
+          [[containers]]
+          type = "$LOCALSTACK_EMULATOR"
+          tag = "$LOCALSTACK_TAG"
+          port = "4566"
+          volumes = ["$PWD/dist:/opt/extension-dist:ro"]
+
+          [cli]
+          check_for_update_on_startup = false
+          EOT
+
+          # host variables prefixed with LOCALSTACK_ are forwarded to the emulator by lstk
+          export LOCALSTACK_DEBUG=1
+          DIST_FILE="$(cd dist && ls localstack_baseshift-*.tar.gz)"
+          export LOCALSTACK_EXTENSION_AUTO_INSTALL="/opt/extension-dist/$DIST_FILE"
+          export LOCALSTACK_BASESHIFT_IMAGE=postgres:17
+          export LOCALSTACK_BASESHIFT_CLONE_POSTGRES_HOST_AUTH_METHOD=trust
+          lstk --non-interactive start
+
+          # lstk returns once LocalStack is healthy - wait until the database port is open as well
+          timeout 120 bash -c 'until (echo > /dev/tcp/localhost/5432) 2>/dev/null; do sleep 2; done'
+
+      - name: Run integration tests
+        run: |
+          cd baseshift
+          make test
+
+      - name: Run end-to-end demo
+        if: matrix.emulator == 'snowflake'
+        run: |
+          cd baseshift
+          make demo
+
+      - name: Print logs
+        if: always()
+        run: |
+          lstk logs --verbose
+          lstk stop

README.md

@@ -68,6 +68,7 @@ You can install the respective extension by calling `localstack extensions insta
 | Extension                                                                                          | Install name | Version | Support status |
 |----------------------------------------------------------------------------------------------------| ------------ |---------| -------------- |
 | [AWS Proxy](https://github.com/localstack/localstack-extensions/tree/main/aws-proxy)               | localstack-extension-aws-proxy | 0.2.1   | Experimental |
+| [Baseshift](https://github.com/localstack/localstack-extensions/tree/main/baseshift)               | localstack-baseshift | 0.1.0   | Experimental |
 | [Diagnosis Viewer](https://github.com/localstack/localstack-extensions/tree/main/diagnosis-viewer) | localstack-extension-diagnosis-viewer | 0.1.0   | Stable |
 | [Hello World](https://github.com/localstack/localstack-extensions/tree/main/hello-world)           | localstack-extension-hello-world | 0.1.0   | Stable |
 | [httpbin](https://github.com/localstack/localstack-extensions/tree/main/httpbin)                   | localstack-extension-httpbin | 0.1.0   | Stable |

baseshift/.gitignore

@@ -0,0 +1,8 @@
+.venv
+dist
+build
+**/*.egg-info
+.eggs
+.pytest_cache
+__pycache__
+demo/baseshift-selfhosted/.env

baseshift/Makefile

@@ -0,0 +1,51 @@
+VENV_BIN = python3 -m venv
+VENV_DIR ?= .venv
+VENV_ACTIVATE = $(VENV_DIR)/bin/activate
+VENV_RUN = . $(VENV_ACTIVATE)
+TEST_PATH ?= tests
+
+usage:          ## Shows usage for this Makefile
+	@cat Makefile | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}'
+
+venv: $(VENV_ACTIVATE)
+
+$(VENV_ACTIVATE): pyproject.toml
+	test -d .venv || $(VENV_BIN) .venv
+	$(VENV_RUN); pip install --upgrade pip setuptools plux
+	$(VENV_RUN); pip install -e .[dev]
+	touch $(VENV_DIR)/bin/activate
+
+clean:
+	rm -rf .venv/
+	rm -rf build/
+	rm -rf .eggs/
+	rm -rf *.egg-info/
+
+install: venv   ## Install dependencies
+	$(VENV_RUN); python -m plux entrypoints
+
+dist: venv      ## Create distribution
+	$(VENV_RUN); python -m build
+
+publish: clean-dist venv dist   ## Publish extension to pypi
+	$(VENV_RUN); pip install --upgrade twine; twine upload dist/*
+
+entrypoints: venv   # Generate plugin entrypoints for Python package
+	$(VENV_RUN); python -m plux entrypoints
+
+format:		    ## Run ruff to format the whole codebase
+	$(VENV_RUN); python -m ruff format .; python -m ruff check --output-format=full --fix .
+
+lint:		    ## Run ruff to lint the codebase
+	$(VENV_RUN); python -m ruff check --output-format=full .
+
+test:		    ## Run integration tests (requires LocalStack running with the Extension installed)
+	$(VENV_RUN); pytest $(PYTEST_ARGS) $(TEST_PATH)
+
+demo: venv	    ## Run the end-to-end demo (requires LocalStack with the Snowflake emulator and the Extension installed)
+	$(VENV_RUN); pip install -q -r demo/requirements.txt; python demo/demo.py all
+
+clean-dist: clean
+	rm -rf dist/
+
+.PHONY: clean clean-dist demo dist install publish usage venv format lint test

baseshift/README.md

@@ -0,0 +1,216 @@
+# Baseshift on LocalStack
+
+This repo contains a [LocalStack Extension](https://github.com/localstack/localstack-extensions) that runs a [Baseshift](https://baseshift.com) database clone next to LocalStack.
+
+Baseshift creates masked, writable clones of your production database (PostgreSQL or MySQL) for development and testing. With this extension, your application code running in LocalStack (e.g., Lambda functions or ECS tasks) can work against a realistic, anonymized copy of your data, all on your local machine.
+
+For PostgreSQL clones, the database is available through the LocalStack gateway (`localhost.localstack.cloud:4566`), as well as on the regular host port `5432`.
+
+## Architecture
+
+```mermaid
+flowchart LR
+    subgraph source["1 · Production data"]
+        rds[("LocalStack RDS<br/>PostgreSQL with PII")]
+    end
+
+    subgraph baseshift["2 · Baseshift (self-hosted)"]
+        direction TB
+        cloud["Baseshift Cloud<br/>control plane"]
+        connector["Connector<br/>masking policy"]
+        repserver["Replication server<br/>masked replica, snapshots"]
+        cloud -.- repserver
+        connector -- "masked data" --> repserver
+    end
+
+    subgraph registry["3 · Snapshot registry"]
+        ecr[("LocalStack ECR<br/>Docker snapshot images")]
+    end
+
+    subgraph clones["4 · Local clones (this extension)"]
+        direction TB
+        extension["Baseshift extension<br/>clones API"]
+        clone1[("Clone 'default'<br/>:5432")]
+        clone2[("Clone 'pr-123'<br/>:15432")]
+        extension -- "start / stop" --> clone1
+        extension -- "start / stop" --> clone2
+    end
+
+    gateway["LocalStack gateway :4566<br/>PostgreSQL routing"]
+
+    subgraph consumers["5 · Consumers of masked data"]
+        direction TB
+        app["App code<br/>Lambda, ECS, ..."]
+        dev["Developer / CI<br/>psql, IDE, tests"]
+        subgraph analytics["Analytics pipeline"]
+            direction LR
+            elt["ELT job"] -- "CSV" --> s3[("LocalStack S3")] -- "COPY INTO" --> snowflake[("LocalStack<br/>Snowflake")]
+        end
+    end
+
+    rds -- "replicate" --> connector
+    repserver -- "push" --> ecr
+    ecr -- "pull" --> extension
+    clone1 --> gateway
+    gateway -- "SQL" --> app
+    gateway -- "SQL" --> dev
+    gateway -- "SQL" --> elt
+    clone2 -. "SQL via host port" .-> dev
+```
+
+1. **Production data**: the source database, e.g. PostgreSQL in (LocalStack) RDS, containing PII.
+2. **Baseshift** replicates the source database via its connector, which applies the masking policy, and the replication server maintains a masked replica and creates snapshots. The components are managed via the Baseshift Cloud control plane. (In the [demo](demo/), a stand-in script currently takes the place of this step.)
+3. The replication server publishes Docker **snapshot images** to a registry, e.g. AWS ECR, or the LocalStack ECR registry for a fully local setup.
+4. This **extension** starts clones from these images as containers next to LocalStack: a default clone at startup (`BASESHIFT_IMAGE`), and further clones on demand via the clones API. Each clone is a writable copy of the masked database.
+5. **Consumers** work with the masked data only: app code running in LocalStack and developer tools connect through the LocalStack gateway (the first PostgreSQL clone is detected by its protocol handshake on port 4566), or to the host port of a clone. An analytics pipeline can extract data from a clone, and load it into Snowflake (see the [demo](demo/)).
+
+## Prerequisites
+
+- Docker
+- LocalStack Pro (free trial available)
+- [`lstk`](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) CLI (`npm install -g @localstack/lstk`)
+- A Baseshift account with a Dub, and a Docker snapshot (clone image) of it
+- AWS CLI, for pulling the clone image from ECR
+- `make`
+
+## Getting the clone image
+
+Baseshift clone images are private to your organization, and are stored in an AWS ECR repository. In the Baseshift dashboard, select your Dub and click **Get docker snapshot** to get the commands for authenticating to ECR and pulling the image, for example:
+
+```bash
+aws ecr get-login-password --region <region> | \
+  docker login --username AWS --password-stdin <account-id>.dkr.ecr.<region>.amazonaws.com
+docker pull <account-id>.dkr.ecr.<region>.amazonaws.com/<repository>:latest
+```
+
+The image needs to be pulled on the host **before** starting LocalStack, as the extension does not authenticate against ECR. Note that ECR logins expire after 12 hours.
+
+See the [Baseshift docs on local clones](https://docs.baseshift.com/docs/clones/local-clones) for more details.
+
+## Install from GitHub repository
+
+`lstk` does not have commands for managing extensions, but LocalStack can install the extension at startup via the `EXTENSION_AUTO_INSTALL` config variable:
+
+```bash
+LOCALSTACK_EXTENSION_AUTO_INSTALL="git+https://github.com/localstack/localstack-extensions.git#egg=localstack-baseshift&subdirectory=baseshift" \
+LOCALSTACK_BASESHIFT_IMAGE=<account-id>.dkr.ecr.<region>.amazonaws.com/<repository>:latest \
+  lstk start
+```
+
+Alternatively, if you are using the legacy `localstack` CLI:
+
+```bash
+localstack extensions install "git+https://github.com/localstack/localstack-extensions.git#egg=localstack-baseshift&subdirectory=baseshift"
+```
+
+## Install local development version
+
+To install the extension into LocalStack in developer mode, you will need Python 3.11, and create a virtual environment in the extensions project.
+
+In the newly generated project, simply run
+
+```bash
+make install
+```
+
+Developer mode currently requires the legacy `localstack` CLI, as `lstk` does not support extensions yet. To enable the extension for LocalStack, run
+
+```bash
+localstack extensions dev enable .
+```
+
+You can then start LocalStack with `EXTENSION_DEV_MODE=1` to load all enabled extensions (the `localstack` CLI mounts the extension sources into the container):
+
+```bash
+EXTENSION_DEV_MODE=1 LOCALSTACK_BASESHIFT_IMAGE=<clone-image> localstack start
+```
+
+## Usage
+
+### Default clone
+
+Start LocalStack with `BASESHIFT_IMAGE` pointing to your clone image, and (if one was configured when creating the Dub) the encryption password:
+
+```bash
+LOCALSTACK_BASESHIFT_IMAGE=<account-id>.dkr.ecr.<region>.amazonaws.com/<repository>:latest \
+LOCALSTACK_BASESHIFT_ENCRYPTION_PASSWORD=<encryption-password> \
+  lstk start
+```
+
+The clone (named `default`) is started in the background once LocalStack is ready. Connect to it, for example with `psql`. Local clones do not require a password; use one of the database users replicated from your source database:
+
+```bash
+# through the LocalStack gateway
+psql -h localhost.localstack.cloud -p 4566 -U <user> <database>
+
+# or directly on the host port
+psql -h localhost -p 5432 -U <user> <database>
+```
+
+From inside LocalStack (e.g., Lambda functions), connect to `localhost.localstack.cloud:4566`.
+
+### Clones API
+
+Clones can also be started and stopped on demand (e.g., one clone per pull request or coding agent), via the API at `http://baseshift.localhost.localstack.cloud:4566/clones`:
+
+```bash
+# start a clone
+curl -X POST http://baseshift.localhost.localstack.cloud:4566/clones \
+  -d '{"name": "pr-123", "image": "<clone-image>"}'
+
+# list clones, with their status and endpoints
+curl http://baseshift.localhost.localstack.cloud:4566/clones
+
+# get / stop a clone
+curl http://baseshift.localhost.localstack.cloud:4566/clones/pr-123
+curl -X DELETE http://baseshift.localhost.localstack.cloud:4566/clones/pr-123
+```
+
+The request accepts `name` (lowercase letters, digits, and dashes), `image`, and optionally `dbType` (`postgres` or `mysql`) and `env` (additional environment variables for the clone container). Clones start asynchronously - their `status` changes from `starting` to `running` (or `failed`, with an `error` message).
+
+Each clone gets its own host port. The first PostgreSQL clone gets port `5432` and is also available through the LocalStack gateway; further clones get ports from `15432` upwards (see the `endpoints` in the API response).
+
+### Clone images in LocalStack ECR
+
+Clone images can also be served by the LocalStack ECR registry, e.g., to share snapshot images within a team, or to emulate the full Baseshift flow locally:
+
+```bash
+lstk aws ecr create-repository --repository-name baseshift/my-dub
+docker tag <clone-image> 000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:4566/baseshift/my-dub:latest
+docker push 000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:4566/baseshift/my-dub:latest
+
+curl -X POST http://baseshift.localhost.localstack.cloud:4566/clones \
+  -d '{"name": "my-dub", "image": "000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:4566/baseshift/my-dub:latest"}'
+```
+
+The LocalStack ECR registry does not require a `docker login`.
+
+### MySQL clones
+
+Set `BASESHIFT_DB_TYPE=mysql` (or `"dbType": "mysql"` in the API) for MySQL clones. MySQL clones are only available on their host port (`3306` for the first one): unlike PostgreSQL, the MySQL protocol starts with the server sending a greeting, so MySQL connections cannot be told apart from other traffic on the gateway port `4566`.
+
+### Environment Variables
+
+- `BASESHIFT_IMAGE`: Image of the default clone to start once LocalStack is ready (optional, clones can also be started via the API)
+- `BASESHIFT_DB_TYPE`: Database engine of the default clone, `postgres` (default) or `mysql`
+- `BASESHIFT_ENCRYPTION_PASSWORD`: Encryption password defined when the Dub was created (passed to the clones as `PASSWORD`)
+- `BASESHIFT_CLONE_<NAME>`: Passed to the clone containers as `<NAME>`, for the [advanced clone options](https://docs.baseshift.com/docs/clones/local-clones#advanced-configuration), e.g. `BASESHIFT_CLONE_BACKUP_SCHEDULE`, `BASESHIFT_CLONE_MAX_BACKUPS`, or `BASESHIFT_CLONE_SPACE_USAGE_MIN_PERCENT`
+
+Note: When starting LocalStack via `lstk` (or the `localstack` CLI), prefix environment variables with `LOCALSTACK_` to forward them to the container, e.g. `LOCALSTACK_BASESHIFT_IMAGE`.
+
+### Limitations
+
+- PostgreSQL connections on the gateway port are detected by their protocol handshake. Running this extension together with another extension that serves PostgreSQL on the gateway (e.g., ParadeDB) is not supported.
+- The clone ports are published on the host, so they must not be in use by another database.
+
+## Demo
+
+The [`demo/`](demo/) directory contains an end-to-end demo of masked production data flowing through a local pipeline: a "production" database in LocalStack RDS, a masked snapshot image in LocalStack ECR, a clone started via this extension, and an ELT pipeline into the LocalStack Snowflake emulator.
+
+## Change Log
+
+- `0.1.0`: Initial release of the extension
+
+## License
+
+This project is licensed under the Apache License, Version 2.0.