feat(docworker, tdk): Extract templating library and support render

.cspell/dictionary.txt

@@ -5,6 +5,8 @@ appconfigdata
 Stubber
 autouse
 caplog
+capfd
+keepends
 delenv
 SIGABRT
 pgconn
@@ -25,6 +27,8 @@ pagebreak
 pagebreaks
 pagebreakpy
 panflute
+Pango
+OOXML
 idstring
 privkey
 searchpath

.github/workflows/pipeline.yml

@@ -134,6 +134,12 @@ jobs:
       run: |
         uv run --no-sync bash scripts/test-package.sh dsw-tdk
 
+    - name: "Test: templating"
+      id: test-templating
+      continue-on-error: true
+      run: |
+        uv run --no-sync bash scripts/test-package.sh dsw-templating
+
     - name: "Evaluate tests"
       if: always()
       shell: bash
@@ -150,8 +156,9 @@ jobs:
           dsw-models=${{ steps.test-models.outcome }}
           dsw-storage=${{ steps.test-storage.outcome }}
           dsw-tdk=${{ steps.test-tdk.outcome }}
+          dsw-templating=${{ steps.test-templating.outcome }}
       run: |
-        EXPECTED=9
+        EXPECTED=10
         SEEN=0
         FAILED=""
         while IFS='=' read -r pkg outcome; do
@@ -175,12 +182,13 @@ jobs:
         fi
         echo "## All $EXPECTED packages passed"
 
-  # dsw-tdk and its only workspace dependency dsw-models declare requires-python
-  # >=3.12 (unlike the rest of the workspace, which is >=3.14) because the TDK is
-  # a CLI that end users pip install. Both are installed standalone from this
-  # checkout here rather than via `uv sync`, which is bound to the root >=3.14
-  # floor. Python 3.14 is already covered across all three OSes by the
-  # test-worker job above.
+  # dsw-tdk, its only workspace dependency dsw-models and dsw-templating (the
+  # rendering library the TDK is meant to use) declare requires-python >=3.12
+  # (unlike the rest of the workspace, which is >=3.14) because the TDK is a CLI
+  # that end users pip install. They are installed standalone from this checkout
+  # here rather than via `uv sync`, which is bound to the root >=3.14 floor.
+  # Python 3.14 is already covered across all three OSes by the test-worker job
+  # above.
   test-tdk-oldest:
     name: Test TDK
     # Avoid double runs: in-repo branches are covered by the push event.
@@ -216,8 +224,8 @@ jobs:
 
     - name: Install dsw-tdk with test dependencies
       run: |
-        # dsw-models from this checkout, not PyPI: dsw-tdk pins the same version
-        uv pip install './packages/dsw-models[test]' './packages/dsw-tdk[test]'
+        # dsw-models and dsw-templating from this checkout, not PyPI: dsw-tdk pins the same version
+        uv pip install './packages/dsw-models[test]' './packages/dsw-templating[all,test]' './packages/dsw-tdk[test]'
         .venv/bin/python --version
 
     - name: "Verify: TDK"
@@ -229,6 +237,11 @@ jobs:
       run: |
         ../../.venv/bin/pytest -s tests
 
+    - name: "Test: Templating"
+      working-directory: packages/dsw-templating
+      run: |
+        ../../.venv/bin/pytest -s tests
+
     - name: "Test: TDK"
       working-directory: packages/dsw-tdk
       run: |
@@ -255,6 +268,7 @@ jobs:
         - dsw-models
         - dsw-storage
         - dsw-tdk
+        - dsw-templating
         python-version:
         - "3.14"
 

.github/workflows/release-package.yml

@@ -108,6 +108,7 @@ jobs:
           - dsw-models
           - dsw-storage
           - dsw-tdk
+          - dsw-templating
 
     steps:
     - name: Check out repository

README.md

@@ -19,8 +19,10 @@ In this monorepo, we manage the following Python packages (each has its own subd
 * [Config (dsw-config)](packages/dsw-config)
 * [Database (dsw-database)](packages/dsw-database)
 * [Storage (dsw-storage)](packages/dsw-storage)
+* [Templating (dsw-templating)](packages/dsw-templating)
 
-Libraries are currently kept compatible with Python 3.14 and higher.
+Libraries are currently kept compatible with Python 3.14 and higher, except for
+`dsw-templating` that renders document templates also for the TDK (Python 3.12 and higher).
 
 ### Utilities
 

packages/dsw-data-seeder/Dockerfile

@@ -67,6 +67,7 @@ COPY packages/dsw-mailer/pyproject.toml /app/packages/dsw-mailer/
 COPY packages/dsw-models/pyproject.toml /app/packages/dsw-models/
 COPY packages/dsw-storage/pyproject.toml /app/packages/dsw-storage/
 COPY packages/dsw-tdk/pyproject.toml /app/packages/dsw-tdk/
+COPY packages/dsw-templating/pyproject.toml /app/packages/dsw-templating/
 
 # project.dependencies is dynamic, so `uv export` must build each member's
 # metadata rather than read it, and hatchling validates project.readme while
@@ -80,6 +81,7 @@ COPY packages/dsw-mailer/README.md /app/packages/dsw-mailer/
 COPY packages/dsw-models/README.md /app/packages/dsw-models/
 COPY packages/dsw-storage/README.md /app/packages/dsw-storage/
 COPY packages/dsw-tdk/README.md /app/packages/dsw-tdk/
+COPY packages/dsw-templating/README.md /app/packages/dsw-templating/
 
 RUN uv export --locked --no-dev --no-emit-workspace --no-hashes --package dsw-data-seeder -o /app/requirements.txt \
  && python -m pip wheel --no-cache-dir --wheel-dir=/app/wheels -r /app/requirements.txt

packages/dsw-document-worker/CHANGELOG.md

@@ -12,7 +12,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - Document template locales: a `.po` file attached to a document template version is applied at render time, so language is a parameter of document generation instead of a property of the format
 - Generation of the POT file with translatable strings (new `generatePotFile` command function), stored in S3 and flagged by `document_template.pot_file_ready`
 - `document.language` and `document.locale` in the document context
-- Translations are available to all steps via `Step.before_render` and the `gettext` / `ngettext` / `pgettext` helpers (see [Translations](./support/Translations.md))
+- Translations are available to all steps via `Step.before_render` and the `gettext` / `ngettext` / `pgettext` helpers (see [Translations](../dsw-templating/support/Translations.md))
 - Lambda handler reads its configuration from AWS AppConfig when `AWS_APP_CONFIG` is set (see `dsw-config`)
 - `docx-landscape.lua` Pandoc filter (a `landscape` div, or `\landscape` / `\portrait` paragraphs, switch DOCX page orientation)
 
@@ -22,6 +22,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - The `jinja2.ext.i18n` extension is always enabled for Jinja-powered steps
 - `extras.project` (and the deprecated `extras.questionnaire`) provide `knowledge_model_package_uuid` instead of `knowledge_package_uuid`
 - The template-facing document context object model (`ctx|to_context_obj`) comes from `dsw-models` (`dsw.models.document_context.graph`) instead of a copy inside the worker; the object model, its attributes and its Markdown rendering are unchanged
+- The rendering engine (formats, steps, Jinja filters and tests, conversions, POT extraction) moved to the new `dsw-templating` package, together with the Pandoc filters, the `pandoc-docx-pagebreakpy` filter (now a command of `dsw-templating[docx]` instead of a separate addon) and the template development documentation; rendering is unchanged
+- Plugins are loaded from the `dsw_templating_plugins` entry point and implement the hooks of `dsw.templating.plugins` (pluggy project `dsw-templating`) instead of `dsw_document_worker_plugins` / `dsw.document_worker.plugins`
+- The Pandoc filters are shipped inside `dsw-templating`; the images no longer copy them to `/pandoc/filters`, which (or `PANDOC_FILTERS`) is still searched first for filters added by a deployment
 
 ### Fixed
 

packages/dsw-document-worker/Dockerfile

@@ -30,7 +30,7 @@ RUN python -m pip wheel --no-deps --wheel-dir=/app/wheels \
       /app/packages/dsw-database \
       /app/packages/dsw-models \
       /app/packages/dsw-storage \
-      /app/packages/dsw-document-worker/addons/* \
+      /app/packages/dsw-templating \
       /app/packages/dsw-document-worker
 
 
@@ -71,6 +71,7 @@ COPY packages/dsw-mailer/pyproject.toml /app/packages/dsw-mailer/
 COPY packages/dsw-models/pyproject.toml /app/packages/dsw-models/
 COPY packages/dsw-storage/pyproject.toml /app/packages/dsw-storage/
 COPY packages/dsw-tdk/pyproject.toml /app/packages/dsw-tdk/
+COPY packages/dsw-templating/pyproject.toml /app/packages/dsw-templating/
 
 # project.dependencies is dynamic, so `uv export` must build each member's
 # metadata rather than read it, and hatchling validates project.readme while
@@ -84,6 +85,7 @@ COPY packages/dsw-mailer/README.md /app/packages/dsw-mailer/
 COPY packages/dsw-models/README.md /app/packages/dsw-models/
 COPY packages/dsw-storage/README.md /app/packages/dsw-storage/
 COPY packages/dsw-tdk/README.md /app/packages/dsw-tdk/
+COPY packages/dsw-templating/README.md /app/packages/dsw-templating/
 
 RUN uv export --locked --no-dev --no-emit-workspace --no-hashes --package dsw-document-worker -o /app/requirements.txt \
  && python -m pip wheel --wheel-dir=/app/wheels -r /app/requirements.txt
@@ -99,10 +101,6 @@ ENV APPLICATION_CONFIG_PATH=/app/config/application.yml \
 COPY packages/dsw-document-worker/resources/fonts /usr/share/fonts/truetype/custom
 RUN fc-cache
 
-# Add Pandoc filters
-COPY packages/dsw-document-worker/resources/pandoc/filters /pandoc/filters
-RUN fc-cache
-
 # Use non-root user
 USER user
 

packages/dsw-document-worker/README.md

@@ -21,12 +21,13 @@ For more information, see [deployment example](https://github.com/ds-wizard/dsw-
 
 For general information, please visit our [User Guide](https://guide.ds-wizard.org).
 
-DSW Document Worker technical documentation for template development:
+Documents are rendered by [dsw-templating](../dsw-templating), which also holds the technical
+documentation for template development:
 
-* [Document Context](./support/DocumentContext.md)
-* [Jinja Filters](./support/JinjaFilters.md)
-* [Jinja Tests](./support/JinjaTests.md)
-* [Translations](./support/Translations.md)
+* [Document Context](../dsw-templating/support/DocumentContext.md)
+* [Jinja Filters](../dsw-templating/support/JinjaFilters.md)
+* [Jinja Tests](../dsw-templating/support/JinjaTests.md)
+* [Translations](../dsw-templating/support/Translations.md)
 
 ## Docker
 

packages/dsw-document-worker/dsw/document_worker/cli.py

@@ -72,8 +72,8 @@ def run(config: DocumentWorkerConfig, workdir: str):
 
 @main.command()
 def list_plugins():
-    from .plugins.manager import create_manager
+    from dsw.templating.plugins import create_manager
 
     pm = create_manager()
-    for plugin in pm.list_name_plugin():
-        click.echo(f'{plugin[0]}: {plugin[1].__name__}')
+    for name, module in pm.list_name_plugin():
+        click.echo(f'{name}: {getattr(module, "__name__", module)}')

packages/dsw-document-worker/dsw/document_worker/consts.py

@@ -8,22 +8,10 @@
 CMD_FUNCTION_GENERATE_POT_FILE = 'generatePotFile'
 COMPONENT_NAME = 'Document Worker'
 DEFAULT_ENCODING = 'utf-8'
-EXIT_SUCCESS = 0
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 PACKAGE_NAME = 'dsw-document-worker'
-PLUGINS_ENTRYPOINT = 'dsw_document_worker_plugins'
 PROG_NAME = 'docworker'
 
-JINJA_EXTENSIONS = ('jinja2.ext.do', 'jinja2.ext.loopcontrols')
-JINJA_FILE_EXTENSIONS = ('.j2', '.jinja', '.jinja2', '.jnj')
-
-# Rendering and POT extraction must agree on this: the msgid of a {% trans %}
-# block depends on it, and the POT file is per document template while a step
-# option would be per format.
-JINJA_I18N_TRIMMED = True
-
-DEFAULT_LANGUAGE = 'en'
-DEFAULT_LOCALE_DOMAIN = 'default'
 LOCALE_PO_FILE_NAME = 'translation.po'
 LOCALE_MO_FILE_NAME = 'translation.mo'
 LOCALE_STAMP_FILE_NAME = 'updated_at'
@@ -46,23 +34,6 @@ class DocumentState:
     FINISHED = 'DoneDocumentState'
 
 
-class TemplateAssetField:
-    UUID = 'uuid'
-    FILENAME = 'fileName'
-    CONTENT_TYPE = 'contentType'
-
-
-class FormatField:
-    UUID = 'uuid'
-    NAME = 'name'
-    STEPS = 'steps'
-
-
-class StepField:
-    NAME = 'name'
-    OPTIONS = 'options'
-
-
 class DocumentNamingStrategy:
     UUID = 'uuid'
     SANITIZE = 'sanitize'

packages/dsw-document-worker/dsw/document_worker/context.py

@@ -66,7 +66,7 @@ def get(cls) -> _Context:
 
     @classmethod
     def initialize(cls, db, s3, config, workdir):
-        from .plugins.manager import create_manager
+        from dsw.templating.plugins import create_manager
         cls._instance = _Context(
             app=AppContext(
                 pm=create_manager(),