feat(config): Support AppConfig for worker configuration

.cspell/dictionary.txt

@@ -1,6 +1,8 @@
 # Python and libraries specific terms
 Aeson
 popleft
+appconfigdata
+Stubber
 autouse
 caplog
 delenv

packages/dsw-config/CHANGELOG.md

@@ -7,6 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- `read_config` loads the configuration YAML from AWS AppConfig when `AWS_APP_CONFIG` is set (environment `Default`, profile derived from the file name, overridable by `AWS_APP_CONFIG_ENVIRONMENT` / `AWS_APP_CONFIG_PROFILE`), and from the local file otherwise; a warm Lambda reuses its session and re-polls only after the poll interval
 
 ## [4.34.0]
 

packages/dsw-config/dsw/config/__init__.py

@@ -1,3 +1,4 @@
+from .appconfig import read_config
 from .parser import (
     DSWConfigParser,
     InvalidConfigurationError,
@@ -9,4 +10,5 @@
     'DSWConfigParser',
     'InvalidConfigurationError',
     'MissingConfigurationError',
+    'read_config',
 ]

packages/dsw-config/dsw/config/appconfig.py

@@ -0,0 +1,121 @@
+"""Read the configuration YAML from AWS AppConfig instead of a local file.
+
+Opt-in: with ``AWS_APP_CONFIG`` unset, the local file is read as before. The
+identifiers follow the backend (``Shared.Bootstrap.AwsAppConfig``): the
+environment is ``Default`` and the profile is the config file name with dots
+replaced by dashes (``application.yml`` -> ``application-yml``); both can be
+overridden. Credentials and region come from the standard AWS chain, i.e. the
+Lambda execution role.
+
+A warm Lambda keeps its session between invocations and only asks AppConfig
+again once the poll interval has passed; an unchanged configuration comes back
+empty and the cached copy is used.
+"""
+from __future__ import annotations
+
+import dataclasses
+import logging
+import os
+import time
+import typing
+
+from .parser import InvalidConfigurationError
+
+
+if typing.TYPE_CHECKING:
+    import pathlib
+
+
+LOG = logging.getLogger(__name__)
+
+VAR_APPLICATION = 'AWS_APP_CONFIG'
+VAR_ENVIRONMENT = 'AWS_APP_CONFIG_ENVIRONMENT'
+VAR_PROFILE = 'AWS_APP_CONFIG_PROFILE'
+DEFAULT_ENVIRONMENT = 'Default'
+
+
+@dataclasses.dataclass
+class _Session:
+    content: str
+    token: str | None
+    interval: int
+    next_poll_at: float
+
+
+_client: typing.Any = None
+_sessions: dict[tuple[str, str, str], _Session] = {}
+
+
+def _get_client():
+    global _client  # noqa: PLW0603
+    if _client is None:
+        import boto3  # noqa: PLC0415
+        _client = boto3.client('appconfigdata')
+    return _client
+
+
+def profile_for(path: pathlib.Path) -> str:
+    return path.name.replace('.', '-')
+
+
+def _start(session_key: tuple[str, str, str]) -> str:
+    application, environment, profile = session_key
+    LOG.info('Starting an AWS AppConfig session (%s)', '/'.join(session_key))
+    start = _get_client().start_configuration_session(
+        ApplicationIdentifier=application,
+        EnvironmentIdentifier=environment,
+        ConfigurationProfileIdentifier=profile,
+    )
+    return start['InitialConfigurationToken']
+
+
+def _poll(session_key: tuple[str, str, str], previous: _Session | None) -> _Session:
+    token = previous.token if previous is not None and previous.token is not None \
+        else _start(session_key)
+    response = _get_client().get_latest_configuration(ConfigurationToken=token)
+    content = response['Configuration'].read().decode('utf-8')
+    if not content:
+        if previous is None:
+            raise InvalidConfigurationError(
+                f'AWS AppConfig returned an empty configuration for {"/".join(session_key)}',
+            )
+        content = previous.content
+    interval = response.get('NextPollIntervalInSeconds', 60)
+    session = _Session(
+        content=content,
+        token=response['NextPollConfigurationToken'],
+        interval=interval,
+        next_poll_at=time.monotonic() + interval,
+    )
+    _sessions[session_key] = session
+    return session
+
+
+def fetch(application: str, environment: str, profile: str) -> str:
+    session_key = (application, environment, profile)
+    session = _sessions.get(session_key)
+    if session is None:
+        return _poll(session_key, None).content
+    if time.monotonic() < session.next_poll_at:
+        return session.content
+    try:
+        return _poll(session_key, session).content
+    except Exception as e:
+        # A token is single-use and expires after 24 hours, and it is spent whether or not
+        # the response reached us. Dropping it makes the next poll start a new session
+        # instead of retrying a dead token forever; the cached content stays the fallback.
+        LOG.warning('AWS AppConfig poll failed, using the cached configuration: %s', e)
+        session.token = None
+        session.next_poll_at = time.monotonic() + session.interval
+        return session.content
+
+
+def read_config(path: pathlib.Path, encoding: str = 'utf-8') -> str:
+    application = os.environ.get(VAR_APPLICATION)
+    if not application:
+        return path.read_text(encoding=encoding)
+    return fetch(
+        application=application,
+        environment=os.environ.get(VAR_ENVIRONMENT) or DEFAULT_ENVIRONMENT,
+        profile=os.environ.get(VAR_PROFILE) or profile_for(path),
+    )

packages/dsw-config/pyproject.toml

@@ -17,6 +17,7 @@ classifiers = [
 ]
 requires-python = ">=3.14, <4"
 dependencies = [
+    "boto3",
     "PyYAML",
     "sentry-sdk",
 ]

packages/dsw-config/tests/test_appconfig.py

@@ -0,0 +1,140 @@
+import io
+import pathlib
+
+import boto3
+import pytest
+from botocore.response import StreamingBody
+from botocore.stub import Stubber
+
+from dsw.config import InvalidConfigurationError, appconfig, read_config
+
+
+SESSION_KEY = ('fw-staging-lambda', 'Default', 'application-yml')
+
+START = {
+    'ApplicationIdentifier': 'fw-staging-lambda',
+    'EnvironmentIdentifier': 'Default',
+    'ConfigurationProfileIdentifier': 'application-yml',
+}
+
+
+def _body(content: bytes) -> StreamingBody:
+    return StreamingBody(io.BytesIO(content), len(content))
+
+
+def _latest(content: bytes, next_token: str, interval: int = 60) -> dict:
+    return {
+        'Configuration': _body(content),
+        'NextPollConfigurationToken': next_token,
+        'NextPollIntervalInSeconds': interval,
+    }
+
+
+@pytest.fixture
+def stubber(monkeypatch):
+    client = boto3.client(
+        'appconfigdata',
+        region_name='eu-central-1',
+        aws_access_key_id='test',
+        aws_secret_access_key='test',  # noqa: S106
+    )
+    monkeypatch.setattr(appconfig, '_client', client)
+    monkeypatch.setattr(appconfig, '_sessions', {})
+    for var in (appconfig.VAR_APPLICATION, appconfig.VAR_ENVIRONMENT, appconfig.VAR_PROFILE):
+        monkeypatch.delenv(var, raising=False)
+    with Stubber(client) as s:
+        yield s
+        s.assert_no_pending_responses()
+
+
+@pytest.fixture
+def config_file(tmp_path) -> pathlib.Path:
+    path = tmp_path / 'application.yml'
+    path.write_text('general:\n  environment: local\n', encoding='utf-8')
+    return path
+
+
+def test_profile_for():
+    assert appconfig.profile_for(pathlib.Path('/var/task/application.yml')) == 'application-yml'
+
+
+def test_reads_local_file_without_app_config(stubber, config_file):
+    assert read_config(config_file) == 'general:\n  environment: local\n'
+
+
+def test_reads_from_app_config(stubber, config_file, monkeypatch):
+    monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda')
+    stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START)
+    stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1'), {'ConfigurationToken': 't0'})
+
+    assert read_config(config_file) == 'remote: 1\n'
+
+
+def test_overrides_environment_and_profile(stubber, config_file, monkeypatch):
+    monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda')
+    monkeypatch.setenv(appconfig.VAR_ENVIRONMENT, 'Other')
+    monkeypatch.setenv(appconfig.VAR_PROFILE, 'mailer-wizard')
+    stubber.add_response(
+        'start_configuration_session',
+        {'InitialConfigurationToken': 't0'},
+        {**START, 'EnvironmentIdentifier': 'Other', 'ConfigurationProfileIdentifier': 'mailer-wizard'},
+    )
+    stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1'), {'ConfigurationToken': 't0'})
+
+    assert read_config(config_file) == 'remote: 1\n'
+
+
+def test_warm_call_within_interval_uses_cache(stubber, config_file, monkeypatch):
+    monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda')
+    stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START)
+    stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1'), {'ConfigurationToken': 't0'})
+
+    read_config(config_file)
+    assert read_config(config_file) == 'remote: 1\n'
+
+
+def test_poll_after_interval_keeps_unchanged_and_picks_up_changes(stubber, config_file, monkeypatch):
+    monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda')
+    stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START)
+    stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1', 0), {'ConfigurationToken': 't0'})
+    stubber.add_response('get_latest_configuration', _latest(b'', 't2', 0), {'ConfigurationToken': 't1'})
+    stubber.add_response('get_latest_configuration', _latest(b'remote: 2\n', 't3', 0), {'ConfigurationToken': 't2'})
+
+    assert read_config(config_file) == 'remote: 1\n'
+    assert read_config(config_file) == 'remote: 1\n'
+    assert read_config(config_file) == 'remote: 2\n'
+
+
+def test_failed_poll_keeps_cached_config(stubber, config_file, monkeypatch):
+    monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda')
+    stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START)
+    stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1', 0), {'ConfigurationToken': 't0'})
+    stubber.add_client_error('get_latest_configuration', 'InternalServerException')
+
+    assert read_config(config_file) == 'remote: 1\n'
+    assert read_config(config_file) == 'remote: 1\n'
+    # The failed poll spent the token, so it is dropped rather than retried
+    assert appconfig._sessions[SESSION_KEY].token is None
+
+
+def test_expired_token_starts_a_new_session(stubber, config_file, monkeypatch):
+    monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda')
+    stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START)
+    stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1', 0), {'ConfigurationToken': 't0'})
+    # A token is valid for 24 hours and only once; an idle warm Lambda hits this
+    stubber.add_client_error('get_latest_configuration', 'BadRequestException')
+    stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't2'}, START)
+    stubber.add_response('get_latest_configuration', _latest(b'remote: 2\n', 't3', 0), {'ConfigurationToken': 't2'})
+
+    assert read_config(config_file) == 'remote: 1\n'
+    assert read_config(config_file) == 'remote: 1\n'
+    assert read_config(config_file) == 'remote: 2\n'
+
+
+def test_empty_initial_config_is_an_error(stubber, config_file, monkeypatch):
+    monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda')
+    stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START)
+    stubber.add_response('get_latest_configuration', _latest(b'', 't1'), {'ConfigurationToken': 't0'})
+
+    with pytest.raises(InvalidConfigurationError):
+        read_config(config_file)

packages/dsw-data-seeder/CHANGELOG.md

@@ -7,6 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- Lambda handler reads its configuration from AWS AppConfig when `AWS_APP_CONFIG` is set (see `dsw-config`)
 
 ## [4.34.0]
 

packages/dsw-data-seeder/dsw/data_seeder/handlers.py

@@ -5,6 +5,8 @@
 import pathlib
 import sys
 
+from dsw.config import read_config
+
 from . import consts
 from .cli import load_config_str
 from .seeder import DataSeeder, SentryReporter
@@ -22,7 +24,7 @@ def lambda_handler(event, context):
         LOG.error('Error: Missing recipe name (environment variable %s)', consts.VAR_SEEDER_RECIPE)
         sys.exit(1)
 
-    config = load_config_str(config_path.read_text(encoding=consts.DEFAULT_ENCODING))
+    config = load_config_str(read_config(config_path, encoding=consts.DEFAULT_ENCODING))
     try:
         seeder = DataSeeder(
             cfg=config,

packages/dsw-document-worker/CHANGELOG.md

@@ -13,6 +13,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - Generation of the POT file with translatable strings (new `generatePotFile` command function), stored in S3 and flagged by `document_template.pot_file_ready`
 - `document.language` and `document.locale` in the document context
 - Translations are available to all steps via `Step.before_render` and the `gettext` / `ngettext` / `pgettext` helpers (see [Translations](./support/Translations.md))
+- Lambda handler reads its configuration from AWS AppConfig when `AWS_APP_CONFIG` is set (see `dsw-config`)
+- `docx-landscape.lua` Pandoc filter (a `landscape` div, or `\landscape` / `\portrait` paragraphs, switch DOCX page orientation)
 
 ### Changed
 

packages/dsw-document-worker/dsw/document_worker/handlers.py

@@ -4,6 +4,8 @@
 import pathlib
 import tempfile
 
+from dsw.config import read_config
+
 from . import consts
 from .cli import load_config_str
 from .worker import DocumentWorker, SentryReporter
@@ -19,7 +21,7 @@ def lambda_handler(event, context):
         os.environ['XDG_CACHE_HOME'] = cache_dir.as_posix()
         fontconfig_tmp.mkdir(parents=True, exist_ok=True)
 
-        config = load_config_str(config_path.read_text(encoding=consts.DEFAULT_ENCODING))
+        config = load_config_str(read_config(config_path, encoding=consts.DEFAULT_ENCODING))
         try:
             doc_worker = DocumentWorker(config, workdir_path)
             doc_worker.run_once()

packages/dsw-document-worker/resources/pandoc/filters/docx-landscape.lua

@@ -0,0 +1,36 @@
+local ooxml = function (s)
+  return pandoc.RawBlock('openxml', s)
+end
+
+local end_portrait_section = ooxml [[
+  <w:p><w:pPr><w:sectPr></w:sectPr></w:pPr></w:p>
+]]
+
+local end_landscape_section = ooxml [[
+<w:p>
+  <w:pPr>
+    <w:sectPr>
+      <w:pgSz w:h="11906" w:w="16838" w:orient="landscape" />
+    </w:sectPr>
+  </w:pPr>
+</w:p>
+]]
+
+function Div (div)
+  if div.classes:includes 'landscape' then
+    div.content:insert(1, end_portrait_section)
+    div.content:insert(end_landscape_section)
+    return div
+  end
+end
+
+function Para (para)
+  for _, inline in ipairs(para.content) do
+    if inline.t == "Str" and inline.text == "\\landscape" then
+      return end_portrait_section
+    end
+    if inline.t == "Str" and inline.text == "\\portrait" then
+      return end_landscape_section
+    end
+  end
+end