diff --git a/.cspell/dictionary.txt b/.cspell/dictionary.txt index 0e86b9ac..b53c1366 100644 --- a/.cspell/dictionary.txt +++ b/.cspell/dictionary.txt @@ -1,6 +1,8 @@ # Python and libraries specific terms Aeson popleft +appconfigdata +Stubber autouse caplog delenv diff --git a/packages/dsw-config/CHANGELOG.md b/packages/dsw-config/CHANGELOG.md index 82466437..e2123407 100644 --- a/packages/dsw-config/CHANGELOG.md +++ b/packages/dsw-config/CHANGELOG.md @@ -7,6 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- `read_config` loads the configuration YAML from AWS AppConfig when `AWS_APP_CONFIG` is set (environment `Default`, profile derived from the file name, overridable by `AWS_APP_CONFIG_ENVIRONMENT` / `AWS_APP_CONFIG_PROFILE`), and from the local file otherwise; a warm Lambda reuses its session and re-polls only after the poll interval ## [4.34.0] diff --git a/packages/dsw-config/dsw/config/__init__.py b/packages/dsw-config/dsw/config/__init__.py index 1f9284f4..e12d94f3 100644 --- a/packages/dsw-config/dsw/config/__init__.py +++ b/packages/dsw-config/dsw/config/__init__.py @@ -1,3 +1,4 @@ +from .appconfig import read_config from .parser import ( DSWConfigParser, InvalidConfigurationError, @@ -9,4 +10,5 @@ 'DSWConfigParser', 'InvalidConfigurationError', 'MissingConfigurationError', + 'read_config', ] diff --git a/packages/dsw-config/dsw/config/appconfig.py b/packages/dsw-config/dsw/config/appconfig.py new file mode 100644 index 00000000..11434f29 --- /dev/null +++ b/packages/dsw-config/dsw/config/appconfig.py @@ -0,0 +1,121 @@ +"""Read the configuration YAML from AWS AppConfig instead of a local file. + +Opt-in: with ``AWS_APP_CONFIG`` unset, the local file is read as before. The +identifiers follow the backend (``Shared.Bootstrap.AwsAppConfig``): the +environment is ``Default`` and the profile is the config file name with dots +replaced by dashes (``application.yml`` -> ``application-yml``); both can be +overridden. Credentials and region come from the standard AWS chain, i.e. the +Lambda execution role. + +A warm Lambda keeps its session between invocations and only asks AppConfig +again once the poll interval has passed; an unchanged configuration comes back +empty and the cached copy is used. +""" +from __future__ import annotations + +import dataclasses +import logging +import os +import time +import typing + +from .parser import InvalidConfigurationError + + +if typing.TYPE_CHECKING: + import pathlib + + +LOG = logging.getLogger(__name__) + +VAR_APPLICATION = 'AWS_APP_CONFIG' +VAR_ENVIRONMENT = 'AWS_APP_CONFIG_ENVIRONMENT' +VAR_PROFILE = 'AWS_APP_CONFIG_PROFILE' +DEFAULT_ENVIRONMENT = 'Default' + + +@dataclasses.dataclass +class _Session: + content: str + token: str | None + interval: int + next_poll_at: float + + +_client: typing.Any = None +_sessions: dict[tuple[str, str, str], _Session] = {} + + +def _get_client(): + global _client # noqa: PLW0603 + if _client is None: + import boto3 # noqa: PLC0415 + _client = boto3.client('appconfigdata') + return _client + + +def profile_for(path: pathlib.Path) -> str: + return path.name.replace('.', '-') + + +def _start(session_key: tuple[str, str, str]) -> str: + application, environment, profile = session_key + LOG.info('Starting an AWS AppConfig session (%s)', '/'.join(session_key)) + start = _get_client().start_configuration_session( + ApplicationIdentifier=application, + EnvironmentIdentifier=environment, + ConfigurationProfileIdentifier=profile, + ) + return start['InitialConfigurationToken'] + + +def _poll(session_key: tuple[str, str, str], previous: _Session | None) -> _Session: + token = previous.token if previous is not None and previous.token is not None \ + else _start(session_key) + response = _get_client().get_latest_configuration(ConfigurationToken=token) + content = response['Configuration'].read().decode('utf-8') + if not content: + if previous is None: + raise InvalidConfigurationError( + f'AWS AppConfig returned an empty configuration for {"/".join(session_key)}', + ) + content = previous.content + interval = response.get('NextPollIntervalInSeconds', 60) + session = _Session( + content=content, + token=response['NextPollConfigurationToken'], + interval=interval, + next_poll_at=time.monotonic() + interval, + ) + _sessions[session_key] = session + return session + + +def fetch(application: str, environment: str, profile: str) -> str: + session_key = (application, environment, profile) + session = _sessions.get(session_key) + if session is None: + return _poll(session_key, None).content + if time.monotonic() < session.next_poll_at: + return session.content + try: + return _poll(session_key, session).content + except Exception as e: + # A token is single-use and expires after 24 hours, and it is spent whether or not + # the response reached us. Dropping it makes the next poll start a new session + # instead of retrying a dead token forever; the cached content stays the fallback. + LOG.warning('AWS AppConfig poll failed, using the cached configuration: %s', e) + session.token = None + session.next_poll_at = time.monotonic() + session.interval + return session.content + + +def read_config(path: pathlib.Path, encoding: str = 'utf-8') -> str: + application = os.environ.get(VAR_APPLICATION) + if not application: + return path.read_text(encoding=encoding) + return fetch( + application=application, + environment=os.environ.get(VAR_ENVIRONMENT) or DEFAULT_ENVIRONMENT, + profile=os.environ.get(VAR_PROFILE) or profile_for(path), + ) diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index 111c5ae9..e50e30ac 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ ] requires-python = ">=3.14, <4" dependencies = [ + "boto3", "PyYAML", "sentry-sdk", ] diff --git a/packages/dsw-config/tests/test_appconfig.py b/packages/dsw-config/tests/test_appconfig.py new file mode 100644 index 00000000..9209e01d --- /dev/null +++ b/packages/dsw-config/tests/test_appconfig.py @@ -0,0 +1,140 @@ +import io +import pathlib + +import boto3 +import pytest +from botocore.response import StreamingBody +from botocore.stub import Stubber + +from dsw.config import InvalidConfigurationError, appconfig, read_config + + +SESSION_KEY = ('fw-staging-lambda', 'Default', 'application-yml') + +START = { + 'ApplicationIdentifier': 'fw-staging-lambda', + 'EnvironmentIdentifier': 'Default', + 'ConfigurationProfileIdentifier': 'application-yml', +} + + +def _body(content: bytes) -> StreamingBody: + return StreamingBody(io.BytesIO(content), len(content)) + + +def _latest(content: bytes, next_token: str, interval: int = 60) -> dict: + return { + 'Configuration': _body(content), + 'NextPollConfigurationToken': next_token, + 'NextPollIntervalInSeconds': interval, + } + + +@pytest.fixture +def stubber(monkeypatch): + client = boto3.client( + 'appconfigdata', + region_name='eu-central-1', + aws_access_key_id='test', + aws_secret_access_key='test', # noqa: S106 + ) + monkeypatch.setattr(appconfig, '_client', client) + monkeypatch.setattr(appconfig, '_sessions', {}) + for var in (appconfig.VAR_APPLICATION, appconfig.VAR_ENVIRONMENT, appconfig.VAR_PROFILE): + monkeypatch.delenv(var, raising=False) + with Stubber(client) as s: + yield s + s.assert_no_pending_responses() + + +@pytest.fixture +def config_file(tmp_path) -> pathlib.Path: + path = tmp_path / 'application.yml' + path.write_text('general:\n environment: local\n', encoding='utf-8') + return path + + +def test_profile_for(): + assert appconfig.profile_for(pathlib.Path('/var/task/application.yml')) == 'application-yml' + + +def test_reads_local_file_without_app_config(stubber, config_file): + assert read_config(config_file) == 'general:\n environment: local\n' + + +def test_reads_from_app_config(stubber, config_file, monkeypatch): + monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda') + stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START) + stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1'), {'ConfigurationToken': 't0'}) + + assert read_config(config_file) == 'remote: 1\n' + + +def test_overrides_environment_and_profile(stubber, config_file, monkeypatch): + monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda') + monkeypatch.setenv(appconfig.VAR_ENVIRONMENT, 'Other') + monkeypatch.setenv(appconfig.VAR_PROFILE, 'mailer-wizard') + stubber.add_response( + 'start_configuration_session', + {'InitialConfigurationToken': 't0'}, + {**START, 'EnvironmentIdentifier': 'Other', 'ConfigurationProfileIdentifier': 'mailer-wizard'}, + ) + stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1'), {'ConfigurationToken': 't0'}) + + assert read_config(config_file) == 'remote: 1\n' + + +def test_warm_call_within_interval_uses_cache(stubber, config_file, monkeypatch): + monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda') + stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START) + stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1'), {'ConfigurationToken': 't0'}) + + read_config(config_file) + assert read_config(config_file) == 'remote: 1\n' + + +def test_poll_after_interval_keeps_unchanged_and_picks_up_changes(stubber, config_file, monkeypatch): + monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda') + stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START) + stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1', 0), {'ConfigurationToken': 't0'}) + stubber.add_response('get_latest_configuration', _latest(b'', 't2', 0), {'ConfigurationToken': 't1'}) + stubber.add_response('get_latest_configuration', _latest(b'remote: 2\n', 't3', 0), {'ConfigurationToken': 't2'}) + + assert read_config(config_file) == 'remote: 1\n' + assert read_config(config_file) == 'remote: 1\n' + assert read_config(config_file) == 'remote: 2\n' + + +def test_failed_poll_keeps_cached_config(stubber, config_file, monkeypatch): + monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda') + stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START) + stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1', 0), {'ConfigurationToken': 't0'}) + stubber.add_client_error('get_latest_configuration', 'InternalServerException') + + assert read_config(config_file) == 'remote: 1\n' + assert read_config(config_file) == 'remote: 1\n' + # The failed poll spent the token, so it is dropped rather than retried + assert appconfig._sessions[SESSION_KEY].token is None + + +def test_expired_token_starts_a_new_session(stubber, config_file, monkeypatch): + monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda') + stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START) + stubber.add_response('get_latest_configuration', _latest(b'remote: 1\n', 't1', 0), {'ConfigurationToken': 't0'}) + # A token is valid for 24 hours and only once; an idle warm Lambda hits this + stubber.add_client_error('get_latest_configuration', 'BadRequestException') + stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't2'}, START) + stubber.add_response('get_latest_configuration', _latest(b'remote: 2\n', 't3', 0), {'ConfigurationToken': 't2'}) + + assert read_config(config_file) == 'remote: 1\n' + assert read_config(config_file) == 'remote: 1\n' + assert read_config(config_file) == 'remote: 2\n' + + +def test_empty_initial_config_is_an_error(stubber, config_file, monkeypatch): + monkeypatch.setenv(appconfig.VAR_APPLICATION, 'fw-staging-lambda') + stubber.add_response('start_configuration_session', {'InitialConfigurationToken': 't0'}, START) + stubber.add_response('get_latest_configuration', _latest(b'', 't1'), {'ConfigurationToken': 't0'}) + + with pytest.raises(InvalidConfigurationError): + read_config(config_file) diff --git a/packages/dsw-data-seeder/CHANGELOG.md b/packages/dsw-data-seeder/CHANGELOG.md index 225ebf28..db530b41 100644 --- a/packages/dsw-data-seeder/CHANGELOG.md +++ b/packages/dsw-data-seeder/CHANGELOG.md @@ -7,6 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- Lambda handler reads its configuration from AWS AppConfig when `AWS_APP_CONFIG` is set (see `dsw-config`) ## [4.34.0] diff --git a/packages/dsw-data-seeder/dsw/data_seeder/handlers.py b/packages/dsw-data-seeder/dsw/data_seeder/handlers.py index 78bdaa79..ebf3e8b7 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/handlers.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/handlers.py @@ -5,6 +5,8 @@ import pathlib import sys +from dsw.config import read_config + from . import consts from .cli import load_config_str from .seeder import DataSeeder, SentryReporter @@ -22,7 +24,7 @@ def lambda_handler(event, context): LOG.error('Error: Missing recipe name (environment variable %s)', consts.VAR_SEEDER_RECIPE) sys.exit(1) - config = load_config_str(config_path.read_text(encoding=consts.DEFAULT_ENCODING)) + config = load_config_str(read_config(config_path, encoding=consts.DEFAULT_ENCODING)) try: seeder = DataSeeder( cfg=config, diff --git a/packages/dsw-document-worker/CHANGELOG.md b/packages/dsw-document-worker/CHANGELOG.md index 29cfb7c3..903d52e0 100644 --- a/packages/dsw-document-worker/CHANGELOG.md +++ b/packages/dsw-document-worker/CHANGELOG.md @@ -13,6 +13,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Generation of the POT file with translatable strings (new `generatePotFile` command function), stored in S3 and flagged by `document_template.pot_file_ready` - `document.language` and `document.locale` in the document context - Translations are available to all steps via `Step.before_render` and the `gettext` / `ngettext` / `pgettext` helpers (see [Translations](./support/Translations.md)) +- Lambda handler reads its configuration from AWS AppConfig when `AWS_APP_CONFIG` is set (see `dsw-config`) +- `docx-landscape.lua` Pandoc filter (a `landscape` div, or `\landscape` / `\portrait` paragraphs, switch DOCX page orientation) ### Changed diff --git a/packages/dsw-document-worker/dsw/document_worker/handlers.py b/packages/dsw-document-worker/dsw/document_worker/handlers.py index 71a832c6..7e73dc09 100644 --- a/packages/dsw-document-worker/dsw/document_worker/handlers.py +++ b/packages/dsw-document-worker/dsw/document_worker/handlers.py @@ -4,6 +4,8 @@ import pathlib import tempfile +from dsw.config import read_config + from . import consts from .cli import load_config_str from .worker import DocumentWorker, SentryReporter @@ -19,7 +21,7 @@ def lambda_handler(event, context): os.environ['XDG_CACHE_HOME'] = cache_dir.as_posix() fontconfig_tmp.mkdir(parents=True, exist_ok=True) - config = load_config_str(config_path.read_text(encoding=consts.DEFAULT_ENCODING)) + config = load_config_str(read_config(config_path, encoding=consts.DEFAULT_ENCODING)) try: doc_worker = DocumentWorker(config, workdir_path) doc_worker.run_once() diff --git a/packages/dsw-document-worker/resources/pandoc/filters/docx-landscape.lua b/packages/dsw-document-worker/resources/pandoc/filters/docx-landscape.lua new file mode 100644 index 00000000..efe00021 --- /dev/null +++ b/packages/dsw-document-worker/resources/pandoc/filters/docx-landscape.lua @@ -0,0 +1,36 @@ +local ooxml = function (s) + return pandoc.RawBlock('openxml', s) +end + +local end_portrait_section = ooxml [[ + +]] + +local end_landscape_section = ooxml [[ + + + + + + + +]] + +function Div (div) + if div.classes:includes 'landscape' then + div.content:insert(1, end_portrait_section) + div.content:insert(end_landscape_section) + return div + end +end + +function Para (para) + for _, inline in ipairs(para.content) do + if inline.t == "Str" and inline.text == "\\landscape" then + return end_portrait_section + end + if inline.t == "Str" and inline.text == "\\portrait" then + return end_landscape_section + end + end +end diff --git a/packages/dsw-mailer/CHANGELOG.md b/packages/dsw-mailer/CHANGELOG.md index 619ff82a..c4b1e08b 100644 --- a/packages/dsw-mailer/CHANGELOG.md +++ b/packages/dsw-mailer/CHANGELOG.md @@ -7,6 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- Lambda handler reads its configuration from AWS AppConfig when `AWS_APP_CONFIG` is set (see `dsw-config`) ## [4.34.0] diff --git a/packages/dsw-mailer/dsw/mailer/handlers.py b/packages/dsw-mailer/dsw/mailer/handlers.py index b4c188bc..466d1219 100644 --- a/packages/dsw-mailer/dsw/mailer/handlers.py +++ b/packages/dsw-mailer/dsw/mailer/handlers.py @@ -3,6 +3,8 @@ import os import pathlib +from dsw.config import read_config + from . import consts from .cli import load_config_str from .mailer import Mailer, SentryReporter @@ -12,7 +14,7 @@ def lambda_handler(event, context): config_path = pathlib.Path(os.getenv(consts.VAR_APP_CONFIG_PATH, '/var/task/application.yml')) workdir_path = pathlib.Path(os.getenv(consts.VAR_WORKDIR_PATH, '/var/task/templates')) - config = load_config_str(config_path.read_text(encoding=consts.DEFAULT_ENCODING)) + config = load_config_str(read_config(config_path, encoding=consts.DEFAULT_ENCODING)) try: mailer = Mailer(config, workdir_path) mailer.run_once() diff --git a/uv.lock b/uv.lock index e737c286..442b00e2 100644 --- a/uv.lock +++ b/uv.lock @@ -444,12 +444,14 @@ requires-dist = [ name = "dsw-config" source = { editable = "packages/dsw-config" } dependencies = [ + { name = "boto3" }, { name = "pyyaml" }, { name = "sentry-sdk" }, ] [package.metadata] requires-dist = [ + { name = "boto3" }, { name = "pyyaml" }, { name = "sentry-sdk" }, ]