From 22a2f6b14d7bef34ffa695e7bb02a7937a85799e Mon Sep 17 00:00:00 2001 From: haseeb Date: Fri, 7 Aug 2026 21:53:49 +0530 Subject: [PATCH 1/2] build shell-operator-neutron container --- .github/workflows/containers.yaml | 3 + containers/shell-operator-neutron/Dockerfile | 11 + .../hooks/router_flavors.py | 363 ++++++++++++++++++ .../shell-operator-neutron/requirements.txt | 2 + 4 files changed, 379 insertions(+) create mode 100644 containers/shell-operator-neutron/Dockerfile create mode 100755 containers/shell-operator-neutron/hooks/router_flavors.py create mode 100644 containers/shell-operator-neutron/requirements.txt diff --git a/.github/workflows/containers.yaml b/.github/workflows/containers.yaml index 69ac88bb2..33119503f 100644 --- a/.github/workflows/containers.yaml +++ b/.github/workflows/containers.yaml @@ -13,6 +13,7 @@ on: - "containers/ironic-nautobot-client/**" - "containers/ironic-vnc-container/**" - "containers/shell-operator-ironic/**" + - "containers/shell-operator-neutron/**" - "containers/understack-tests/**" - "python/**" - ".github/workflows/containers.yaml" @@ -45,6 +46,8 @@ jobs: prebuild_script_working_dir: containers/ironic-vnc-container/ - name: shell-operator-ironic target: prod + - name: shell-operator-neutron + target: prod - name: nautobot target: prod uses: ./.github/workflows/build-container-reuse.yaml diff --git a/containers/shell-operator-neutron/Dockerfile b/containers/shell-operator-neutron/Dockerfile new file mode 100644 index 000000000..3824c4681 --- /dev/null +++ b/containers/shell-operator-neutron/Dockerfile @@ -0,0 +1,11 @@ +FROM ghcr.io/flant/shell-operator:v1.13.1 AS prod +LABEL org.opencontainers.image.description="shell-operator for Neutron router flavors" + +RUN --mount=type=cache,target=/var/cache/apk apk add python3 +RUN python3 -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +COPY containers/shell-operator-neutron/requirements.txt requirements.txt +RUN pip install --no-cache --upgrade -r requirements.txt + +COPY containers/shell-operator-neutron/hooks /hooks diff --git a/containers/shell-operator-neutron/hooks/router_flavors.py b/containers/shell-operator-neutron/hooks/router_flavors.py new file mode 100755 index 000000000..0726eacef --- /dev/null +++ b/containers/shell-operator-neutron/hooks/router_flavors.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +"""Reconcile Neutron router flavors and service profiles from JSON config.""" + +from __future__ import annotations + +import ast +import json +import os +import sys +import time +from typing import Any + + +HOOK_CONFIG = { + "configVersion": "v1", + "onStartup": 1, + "settings": { + "executionMinInterval": "30s", + "executionBurst": 1, + }, +} + +CONFIG_PATH = os.environ.get( + "NEUTRON_ROUTER_FLAVORS_CONFIG", + "/etc/neutron-router-flavors/router_flavors.json", +) +DEFAULT_SERVICE_TYPE = os.environ.get( + "NEUTRON_ROUTER_FLAVOR_SERVICE_TYPE", + "L3_ROUTER_NAT", +) +READY_RETRIES = int(os.environ.get("NEUTRON_ROUTER_FLAVOR_READY_RETRIES", "30")) +READY_DELAY = float(os.environ.get("NEUTRON_ROUTER_FLAVOR_READY_DELAY", "10")) +_MISSING = object() + + +class ConfigError(Exception): + pass + + +def log(message: str) -> None: + print(f"[router_flavors] {message}", file=sys.stderr) + + +def _resource_value(resource: Any, name: str) -> Any: + if isinstance(resource, dict): + return resource[name] if name in resource else _MISSING + + getter = getattr(resource, "get", None) + if callable(getter): + try: + value = getter(name, _MISSING) + except TypeError: + try: + value = getter(name) + except Exception: + value = _MISSING + except Exception: + value = _MISSING + + if value is not _MISSING: + return value + + value = getattr(resource, name, _MISSING) + if value is not _MISSING: + return value + + try: + data = resource.to_dict(computed=False) + except Exception: + data = {} + + return data[name] if name in data else _MISSING + + +def get_value(resource: Any, *names: str, default: Any = None) -> Any: + for name in names: + value = _resource_value(resource, name) + if value is not _MISSING and value is not None: + return value + + return default + + +def resource_id(resource: Any) -> str: + value = get_value(resource, "id", "ID", "Id") + if not value: + raise RuntimeError(f"Unable to read ID from resource {resource!r}") + return str(value) + + +def normalize_meta_info(value: Any) -> Any: + if value is None or value == "": + return {} + + if isinstance(value, str): + text = value.strip() + if not text: + return {} + + try: + return json.loads(text) + except json.JSONDecodeError: + try: + return ast.literal_eval(text) + except (SyntaxError, ValueError): + return text + + return value + + +def meta_info_payload(value: Any) -> str: + normalized = normalize_meta_info(value) + return json.dumps(normalized, sort_keys=True, separators=(",", ":")) + + +def meta_info_matches(current: Any, desired: Any) -> bool: + return meta_info_payload(current) == meta_info_payload(desired) + + +def is_not_found(exc: Exception) -> bool: + return getattr(exc, "status_code", None) == 404 or exc.__class__.__name__ in { + "NotFoundException", + "ResourceNotFound", + } + + +def is_conflict(exc: Exception) -> bool: + return ( + getattr(exc, "status_code", None) == 409 + or exc.__class__.__name__ in {"ConflictException", "ResourceConflict"} + or "already" in str(exc).lower() + ) + + +def connect_openstack(os_cloud: str | None) -> Any: + try: + import openstack + except ImportError as exc: + raise RuntimeError("openstacksdk is required to run this hook") from exc + + return openstack.connect(cloud=os_cloud) + + +def load_config(path: str) -> list[dict[str, Any]]: + if not os.path.isfile(path): + raise ConfigError(f"Router flavor config not found at {path}") + + with open(path, encoding="utf-8") as config_file: + flavors = json.load(config_file) + + if not isinstance(flavors, list): + raise ConfigError("Router flavor config must be a JSON list") + + return flavors + + +def wait_for_openstack_network(conn: Any) -> None: + for attempt in range(1, READY_RETRIES + 1): + try: + next(iter(conn.network.flavors()), None) + return + except Exception as exc: + if attempt >= READY_RETRIES: + raise RuntimeError( + f"Neutron API did not become ready after {READY_RETRIES} attempt(s)" + ) from exc + + log(f"Waiting for Neutron API ({attempt}/{READY_RETRIES}): {exc}") + time.sleep(READY_DELAY) + + +def get_service_profile(conn: Any, profile_id: str) -> Any | None: + try: + return conn.network.get_service_profile(profile_id) + except Exception as exc: + if is_not_found(exc): + return None + raise + + +def find_matching_profile(conn: Any, driver: str, meta_info: Any) -> Any | None: + for profile in conn.network.service_profiles(): + if get_value(profile, "driver", "Driver", default="") != driver: + continue + + if meta_info_matches( + get_value(profile, "meta_info", default={}), + meta_info, + ): + return profile + + return None + + +def ensure_profile( + conn: Any, + name: str, + driver: str, + description: str, + meta_info: Any, + configured_profile_id: str, +) -> Any: + if configured_profile_id: + profile = get_service_profile(conn, configured_profile_id) + if profile: + log(f"Using configured service profile {configured_profile_id} for {name}") + return profile + + log( + f"Configured service profile {configured_profile_id} " + f"for {name} was not found" + ) + + profile = find_matching_profile(conn, driver, meta_info) + if profile: + profile_id = resource_id(profile) + log(f"Reusing service profile {profile_id} for {name}") + # Neutron rejects service profile updates once they are used by service instances. + # Matching driver/meta_info is enough for idempotent reuse. + return profile + + log(f"Creating service profile for {name} driver={driver}") + return conn.network.create_service_profile( + description=description, + driver=driver, + meta_info=meta_info_payload(meta_info), + is_enabled=True, + ) + + +def find_flavor(conn: Any, name: str) -> Any | None: + for flavor in conn.network.flavors(name=name): + if get_value(flavor, "name", "Name") == name: + return flavor + + return None + + +def ensure_flavor(conn: Any, name: str, service_type: str, description: str) -> Any: + flavor = find_flavor(conn, name) + if flavor: + log(f"Router flavor {name} already exists") + current_description = get_value(flavor, "description", "Description") + if description and current_description != description: + return conn.network.update_flavor(flavor, description=description) + return flavor + + log(f"Creating router flavor {name} service_type={service_type}") + attrs = { + "name": name, + "service_type": service_type, + "is_enabled": True, + } + if description: + attrs["description"] = description + return conn.network.create_flavor(**attrs) + + +def service_profile_ids(flavor: Any) -> list[str]: + profiles = get_value( + flavor, + "service_profile_ids", + "service_profiles", + "profiles", + default=[], + ) + if profiles is None: + return [] + if isinstance(profiles, str): + return [item.strip() for item in profiles.split(",") if item.strip()] + return [str(profile) for profile in profiles] + + +def ensure_profile_attached(conn: Any, flavor: Any, profile: Any) -> Any: + flavor = conn.network.get_flavor(flavor) + flavor_id = resource_id(flavor) + profile_id = resource_id(profile) + + if profile_id in service_profile_ids(flavor): + flavor_name = get_value(flavor, "name", "Name", default=flavor_id) + log(f"Router flavor {flavor_name} already has service profile {profile_id}") + return flavor + + log(f"Binding service profile {profile_id} to router flavor {flavor_id}") + try: + conn.network.associate_flavor_with_service_profile(flavor, profile) + except Exception as exc: + if not is_conflict(exc): + raise + log(f"Router flavor {flavor_id} already has service profile {profile_id}") + + return conn.network.get_flavor(flavor) + + +def render_flavor(flavor: Any) -> dict[str, Any]: + return { + "id": get_value(flavor, "id", "ID"), + "name": get_value(flavor, "name", "Name"), + "service_type": get_value(flavor, "service_type", "Service Type"), + "description": get_value(flavor, "description", "Description"), + "service_profile_ids": service_profile_ids(flavor), + } + + +def config_meta_info(flavor_config: dict[str, Any]) -> Any: + if "metainfo" in flavor_config: + name = flavor_config.get("name", "") + raise ConfigError(f"Router flavor {name} uses metainfo; use meta_info instead") + + return flavor_config.get("meta_info", {}) + + +def sync_flavor(conn: Any, flavor_config: dict[str, Any]) -> None: + name = flavor_config.get("name") + driver = flavor_config.get("driver") + if not name or not driver: + raise ConfigError( + "Each router flavor entry must define name and driver: " f"{flavor_config}" + ) + + description = flavor_config.get("description", "") + profile_description = flavor_config.get("profile_description", description) + service_type = flavor_config.get("service_type", DEFAULT_SERVICE_TYPE) + profile_id = flavor_config.get("profile_id", "") + meta_info = config_meta_info(flavor_config) + + log(f"Reconciling router flavor {name}") + profile = ensure_profile( + conn, + name, + driver, + profile_description, + meta_info, + profile_id, + ) + flavor = ensure_flavor(conn, name, service_type, description) + flavor = ensure_profile_attached(conn, flavor, profile) + print(json.dumps(render_flavor(flavor), sort_keys=True)) + + +def run() -> int: + if len(sys.argv) > 1 and sys.argv[1] == "--config": + print(json.dumps(HOOK_CONFIG, indent=2)) + return 0 + + flavors = load_config(CONFIG_PATH) + conn = connect_openstack(os.environ.get("OS_CLOUD")) + wait_for_openstack_network(conn) + + log(f"Found {len(flavors)} router flavor(s) to reconcile") + for flavor_config in flavors: + sync_flavor(conn, flavor_config) + + log("Finished reconciling router flavors") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(run()) + except Exception as exc: + log(str(exc)) + sys.exit(1) diff --git a/containers/shell-operator-neutron/requirements.txt b/containers/shell-operator-neutron/requirements.txt new file mode 100644 index 000000000..c9d71b957 --- /dev/null +++ b/containers/shell-operator-neutron/requirements.txt @@ -0,0 +1,2 @@ +pip +openstacksdk From cb89df1c74f04b8153a4e296292f8a52a178784d Mon Sep 17 00:00:00 2001 From: haseeb Date: Mon, 10 Aug 2026 18:01:52 +0530 Subject: [PATCH 2/2] restructure shell-operator-neutron --- .dockerignore | 3 + .github/workflows/containers-openstack.yaml | 11 + .github/workflows/containers.yaml | 3 - .../neutron-router-flavor-sync/Dockerfile | 23 + containers/shell-operator-neutron/Dockerfile | 11 - .../hooks/router_flavors.py | 363 -------------- .../shell-operator-neutron/requirements.txt | 2 - python/understack-neutron-flavors/README.md | 4 + .../understack-neutron-flavors/pyproject.toml | 84 ++++ .../tests/__init__.py | 1 + .../tests/test_router_flavors.py | 451 ++++++++++++++++++ .../understack_neutron_flavors/__init__.py | 1 + .../create_router_flavors.py | 113 +++++ .../delete_router_flavors.py | 209 ++++++++ .../router_flavors.py | 58 +++ .../router_flavors_common.py | 309 ++++++++++++ .../update_router_flavors.py | 69 +++ 17 files changed, 1336 insertions(+), 379 deletions(-) create mode 100644 containers/neutron-router-flavor-sync/Dockerfile delete mode 100644 containers/shell-operator-neutron/Dockerfile delete mode 100755 containers/shell-operator-neutron/hooks/router_flavors.py delete mode 100644 containers/shell-operator-neutron/requirements.txt create mode 100644 python/understack-neutron-flavors/README.md create mode 100644 python/understack-neutron-flavors/pyproject.toml create mode 100644 python/understack-neutron-flavors/tests/__init__.py create mode 100644 python/understack-neutron-flavors/tests/test_router_flavors.py create mode 100644 python/understack-neutron-flavors/understack_neutron_flavors/__init__.py create mode 100644 python/understack-neutron-flavors/understack_neutron_flavors/create_router_flavors.py create mode 100644 python/understack-neutron-flavors/understack_neutron_flavors/delete_router_flavors.py create mode 100644 python/understack-neutron-flavors/understack_neutron_flavors/router_flavors.py create mode 100644 python/understack-neutron-flavors/understack_neutron_flavors/router_flavors_common.py create mode 100644 python/understack-neutron-flavors/understack_neutron_flavors/update_router_flavors.py diff --git a/.dockerignore b/.dockerignore index 1d17dae13..8329ed251 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,4 @@ +.idea .venv +__pycache__ +*.py[cod] diff --git a/.github/workflows/containers-openstack.yaml b/.github/workflows/containers-openstack.yaml index d31689eef..f2d37ddf0 100644 --- a/.github/workflows/containers-openstack.yaml +++ b/.github/workflows/containers-openstack.yaml @@ -17,6 +17,7 @@ on: - "containers/octavia/**" - "containers/openstack-client/**" - "containers/placement/**" + - "containers/neutron-router-flavor-sync/**" - "containers/skyline/**" - ".github/workflows/containers-openstack.yaml" - ".github/workflows/build-container-reuse.yaml" @@ -74,6 +75,16 @@ jobs: build_args: OPENSTACK_VERSION=2026.1 latest_name: "2026.1" + neutron-router-flavor-sync: + uses: ./.github/workflows/build-container-reuse.yaml + secrets: inherit + with: + container_name: neutron-router-flavor-sync + dockerfile_path: containers/neutron-router-flavor-sync/Dockerfile + build_args: OPENSTACK_VERSION=2026.1 + latest_name: "2026.1" + target: prod + ironic: uses: ./.github/workflows/build-container-reuse.yaml secrets: inherit diff --git a/.github/workflows/containers.yaml b/.github/workflows/containers.yaml index 33119503f..69ac88bb2 100644 --- a/.github/workflows/containers.yaml +++ b/.github/workflows/containers.yaml @@ -13,7 +13,6 @@ on: - "containers/ironic-nautobot-client/**" - "containers/ironic-vnc-container/**" - "containers/shell-operator-ironic/**" - - "containers/shell-operator-neutron/**" - "containers/understack-tests/**" - "python/**" - ".github/workflows/containers.yaml" @@ -46,8 +45,6 @@ jobs: prebuild_script_working_dir: containers/ironic-vnc-container/ - name: shell-operator-ironic target: prod - - name: shell-operator-neutron - target: prod - name: nautobot target: prod uses: ./.github/workflows/build-container-reuse.yaml diff --git a/containers/neutron-router-flavor-sync/Dockerfile b/containers/neutron-router-flavor-sync/Dockerfile new file mode 100644 index 000000000..c8170889f --- /dev/null +++ b/containers/neutron-router-flavor-sync/Dockerfile @@ -0,0 +1,23 @@ +# syntax=docker/dockerfile:1 + +ARG OPENSTACK_VERSION="required_argument" +FROM ghcr.io/flant/shell-operator:v1.13.1 AS prod +LABEL org.opencontainers.image.description="Neutron router flavor sync shell-operator" + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +RUN --mount=type=cache,target=/var/cache/apk apk add python3 +RUN python3 -m venv /opt/venv +ENV VIRTUAL_ENV="/opt/venv" +ENV PATH="/opt/venv/bin:$PATH" + +ARG OPENSTACK_VERSION="required_argument" +ADD https://releases.openstack.org/constraints/upper/${OPENSTACK_VERSION} /upper-constraints.txt +COPY python/understack-neutron-flavors /src/understack-neutron-flavors +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install \ + --upgrade \ + --constraint /upper-constraints.txt \ + /src/understack-neutron-flavors + +COPY --chmod=755 python/understack-neutron-flavors/understack_neutron_flavors/router_flavors.py /hooks/router_flavors.py diff --git a/containers/shell-operator-neutron/Dockerfile b/containers/shell-operator-neutron/Dockerfile deleted file mode 100644 index 3824c4681..000000000 --- a/containers/shell-operator-neutron/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM ghcr.io/flant/shell-operator:v1.13.1 AS prod -LABEL org.opencontainers.image.description="shell-operator for Neutron router flavors" - -RUN --mount=type=cache,target=/var/cache/apk apk add python3 -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" - -COPY containers/shell-operator-neutron/requirements.txt requirements.txt -RUN pip install --no-cache --upgrade -r requirements.txt - -COPY containers/shell-operator-neutron/hooks /hooks diff --git a/containers/shell-operator-neutron/hooks/router_flavors.py b/containers/shell-operator-neutron/hooks/router_flavors.py deleted file mode 100755 index 0726eacef..000000000 --- a/containers/shell-operator-neutron/hooks/router_flavors.py +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env python3 -"""Reconcile Neutron router flavors and service profiles from JSON config.""" - -from __future__ import annotations - -import ast -import json -import os -import sys -import time -from typing import Any - - -HOOK_CONFIG = { - "configVersion": "v1", - "onStartup": 1, - "settings": { - "executionMinInterval": "30s", - "executionBurst": 1, - }, -} - -CONFIG_PATH = os.environ.get( - "NEUTRON_ROUTER_FLAVORS_CONFIG", - "/etc/neutron-router-flavors/router_flavors.json", -) -DEFAULT_SERVICE_TYPE = os.environ.get( - "NEUTRON_ROUTER_FLAVOR_SERVICE_TYPE", - "L3_ROUTER_NAT", -) -READY_RETRIES = int(os.environ.get("NEUTRON_ROUTER_FLAVOR_READY_RETRIES", "30")) -READY_DELAY = float(os.environ.get("NEUTRON_ROUTER_FLAVOR_READY_DELAY", "10")) -_MISSING = object() - - -class ConfigError(Exception): - pass - - -def log(message: str) -> None: - print(f"[router_flavors] {message}", file=sys.stderr) - - -def _resource_value(resource: Any, name: str) -> Any: - if isinstance(resource, dict): - return resource[name] if name in resource else _MISSING - - getter = getattr(resource, "get", None) - if callable(getter): - try: - value = getter(name, _MISSING) - except TypeError: - try: - value = getter(name) - except Exception: - value = _MISSING - except Exception: - value = _MISSING - - if value is not _MISSING: - return value - - value = getattr(resource, name, _MISSING) - if value is not _MISSING: - return value - - try: - data = resource.to_dict(computed=False) - except Exception: - data = {} - - return data[name] if name in data else _MISSING - - -def get_value(resource: Any, *names: str, default: Any = None) -> Any: - for name in names: - value = _resource_value(resource, name) - if value is not _MISSING and value is not None: - return value - - return default - - -def resource_id(resource: Any) -> str: - value = get_value(resource, "id", "ID", "Id") - if not value: - raise RuntimeError(f"Unable to read ID from resource {resource!r}") - return str(value) - - -def normalize_meta_info(value: Any) -> Any: - if value is None or value == "": - return {} - - if isinstance(value, str): - text = value.strip() - if not text: - return {} - - try: - return json.loads(text) - except json.JSONDecodeError: - try: - return ast.literal_eval(text) - except (SyntaxError, ValueError): - return text - - return value - - -def meta_info_payload(value: Any) -> str: - normalized = normalize_meta_info(value) - return json.dumps(normalized, sort_keys=True, separators=(",", ":")) - - -def meta_info_matches(current: Any, desired: Any) -> bool: - return meta_info_payload(current) == meta_info_payload(desired) - - -def is_not_found(exc: Exception) -> bool: - return getattr(exc, "status_code", None) == 404 or exc.__class__.__name__ in { - "NotFoundException", - "ResourceNotFound", - } - - -def is_conflict(exc: Exception) -> bool: - return ( - getattr(exc, "status_code", None) == 409 - or exc.__class__.__name__ in {"ConflictException", "ResourceConflict"} - or "already" in str(exc).lower() - ) - - -def connect_openstack(os_cloud: str | None) -> Any: - try: - import openstack - except ImportError as exc: - raise RuntimeError("openstacksdk is required to run this hook") from exc - - return openstack.connect(cloud=os_cloud) - - -def load_config(path: str) -> list[dict[str, Any]]: - if not os.path.isfile(path): - raise ConfigError(f"Router flavor config not found at {path}") - - with open(path, encoding="utf-8") as config_file: - flavors = json.load(config_file) - - if not isinstance(flavors, list): - raise ConfigError("Router flavor config must be a JSON list") - - return flavors - - -def wait_for_openstack_network(conn: Any) -> None: - for attempt in range(1, READY_RETRIES + 1): - try: - next(iter(conn.network.flavors()), None) - return - except Exception as exc: - if attempt >= READY_RETRIES: - raise RuntimeError( - f"Neutron API did not become ready after {READY_RETRIES} attempt(s)" - ) from exc - - log(f"Waiting for Neutron API ({attempt}/{READY_RETRIES}): {exc}") - time.sleep(READY_DELAY) - - -def get_service_profile(conn: Any, profile_id: str) -> Any | None: - try: - return conn.network.get_service_profile(profile_id) - except Exception as exc: - if is_not_found(exc): - return None - raise - - -def find_matching_profile(conn: Any, driver: str, meta_info: Any) -> Any | None: - for profile in conn.network.service_profiles(): - if get_value(profile, "driver", "Driver", default="") != driver: - continue - - if meta_info_matches( - get_value(profile, "meta_info", default={}), - meta_info, - ): - return profile - - return None - - -def ensure_profile( - conn: Any, - name: str, - driver: str, - description: str, - meta_info: Any, - configured_profile_id: str, -) -> Any: - if configured_profile_id: - profile = get_service_profile(conn, configured_profile_id) - if profile: - log(f"Using configured service profile {configured_profile_id} for {name}") - return profile - - log( - f"Configured service profile {configured_profile_id} " - f"for {name} was not found" - ) - - profile = find_matching_profile(conn, driver, meta_info) - if profile: - profile_id = resource_id(profile) - log(f"Reusing service profile {profile_id} for {name}") - # Neutron rejects service profile updates once they are used by service instances. - # Matching driver/meta_info is enough for idempotent reuse. - return profile - - log(f"Creating service profile for {name} driver={driver}") - return conn.network.create_service_profile( - description=description, - driver=driver, - meta_info=meta_info_payload(meta_info), - is_enabled=True, - ) - - -def find_flavor(conn: Any, name: str) -> Any | None: - for flavor in conn.network.flavors(name=name): - if get_value(flavor, "name", "Name") == name: - return flavor - - return None - - -def ensure_flavor(conn: Any, name: str, service_type: str, description: str) -> Any: - flavor = find_flavor(conn, name) - if flavor: - log(f"Router flavor {name} already exists") - current_description = get_value(flavor, "description", "Description") - if description and current_description != description: - return conn.network.update_flavor(flavor, description=description) - return flavor - - log(f"Creating router flavor {name} service_type={service_type}") - attrs = { - "name": name, - "service_type": service_type, - "is_enabled": True, - } - if description: - attrs["description"] = description - return conn.network.create_flavor(**attrs) - - -def service_profile_ids(flavor: Any) -> list[str]: - profiles = get_value( - flavor, - "service_profile_ids", - "service_profiles", - "profiles", - default=[], - ) - if profiles is None: - return [] - if isinstance(profiles, str): - return [item.strip() for item in profiles.split(",") if item.strip()] - return [str(profile) for profile in profiles] - - -def ensure_profile_attached(conn: Any, flavor: Any, profile: Any) -> Any: - flavor = conn.network.get_flavor(flavor) - flavor_id = resource_id(flavor) - profile_id = resource_id(profile) - - if profile_id in service_profile_ids(flavor): - flavor_name = get_value(flavor, "name", "Name", default=flavor_id) - log(f"Router flavor {flavor_name} already has service profile {profile_id}") - return flavor - - log(f"Binding service profile {profile_id} to router flavor {flavor_id}") - try: - conn.network.associate_flavor_with_service_profile(flavor, profile) - except Exception as exc: - if not is_conflict(exc): - raise - log(f"Router flavor {flavor_id} already has service profile {profile_id}") - - return conn.network.get_flavor(flavor) - - -def render_flavor(flavor: Any) -> dict[str, Any]: - return { - "id": get_value(flavor, "id", "ID"), - "name": get_value(flavor, "name", "Name"), - "service_type": get_value(flavor, "service_type", "Service Type"), - "description": get_value(flavor, "description", "Description"), - "service_profile_ids": service_profile_ids(flavor), - } - - -def config_meta_info(flavor_config: dict[str, Any]) -> Any: - if "metainfo" in flavor_config: - name = flavor_config.get("name", "") - raise ConfigError(f"Router flavor {name} uses metainfo; use meta_info instead") - - return flavor_config.get("meta_info", {}) - - -def sync_flavor(conn: Any, flavor_config: dict[str, Any]) -> None: - name = flavor_config.get("name") - driver = flavor_config.get("driver") - if not name or not driver: - raise ConfigError( - "Each router flavor entry must define name and driver: " f"{flavor_config}" - ) - - description = flavor_config.get("description", "") - profile_description = flavor_config.get("profile_description", description) - service_type = flavor_config.get("service_type", DEFAULT_SERVICE_TYPE) - profile_id = flavor_config.get("profile_id", "") - meta_info = config_meta_info(flavor_config) - - log(f"Reconciling router flavor {name}") - profile = ensure_profile( - conn, - name, - driver, - profile_description, - meta_info, - profile_id, - ) - flavor = ensure_flavor(conn, name, service_type, description) - flavor = ensure_profile_attached(conn, flavor, profile) - print(json.dumps(render_flavor(flavor), sort_keys=True)) - - -def run() -> int: - if len(sys.argv) > 1 and sys.argv[1] == "--config": - print(json.dumps(HOOK_CONFIG, indent=2)) - return 0 - - flavors = load_config(CONFIG_PATH) - conn = connect_openstack(os.environ.get("OS_CLOUD")) - wait_for_openstack_network(conn) - - log(f"Found {len(flavors)} router flavor(s) to reconcile") - for flavor_config in flavors: - sync_flavor(conn, flavor_config) - - log("Finished reconciling router flavors") - return 0 - - -if __name__ == "__main__": - try: - sys.exit(run()) - except Exception as exc: - log(str(exc)) - sys.exit(1) diff --git a/containers/shell-operator-neutron/requirements.txt b/containers/shell-operator-neutron/requirements.txt deleted file mode 100644 index c9d71b957..000000000 --- a/containers/shell-operator-neutron/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pip -openstacksdk diff --git a/python/understack-neutron-flavors/README.md b/python/understack-neutron-flavors/README.md new file mode 100644 index 000000000..22b3051e2 --- /dev/null +++ b/python/understack-neutron-flavors/README.md @@ -0,0 +1,4 @@ +# understack-neutron-flavors + +Shell-operator hooks for reconciling Neutron router flavors and service +profiles from UnderStack deploy configuration. diff --git a/python/understack-neutron-flavors/pyproject.toml b/python/understack-neutron-flavors/pyproject.toml new file mode 100644 index 000000000..27f3b0a9b --- /dev/null +++ b/python/understack-neutron-flavors/pyproject.toml @@ -0,0 +1,84 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "understack-neutron-flavors" +description = "Shell-operator hooks for UnderStack Neutron reconciliation" +authors = [{ name = "Understack Developers" }] +requires-python = ">=3.12" +readme = "README.md" +dependencies = [ + "openstacksdk>=4.2.0,<5", +] +license = { text = "Apache-2.0" } +dynamic = ["version"] + +[project.scripts] +neutron-router-flavors = "understack_neutron_flavors.router_flavors:main" + +[project.urls] +Homepage = "https://github.com/rackerlabs/understack" +Issues = "https://github.com/rackerlabs/understack/issues" + +[dependency-groups] +test = [ + "pytest<10", + "pytest-github-actions-annotate-failures", + "pytest-cov>=6.1.0", +] + +[tool.uv] +default-groups = ["test"] + +[tool.hatch.build.targets.sdist] +include = ["understack_neutron_flavors"] + +[tool.hatch.build.targets.wheel] +include = ["understack_neutron_flavors"] + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.version.raw-options] +root = "../../" +tag_regex = "^understack-neutron-flavors/v(?P.*)$" +git_describe_command = "git describe --dirty --tags --long --match understack-neutron-flavors/v*" +fallback_version = "0.1.0" + +[tool.ruff] +target-version = "py312" +fix = true + +[tool.ruff.lint] +select = [ + "D", # pydocstyle + "E", # pycodestyle (error) + "F", # pyflakes + "B", # flake8-bugbear + "I", # isort + "S", # flake8-bandit + "UP", # pyupgrade + "ASYNC", # flake8-async +] + +ignore = [ + "D100", # don't require docs for every module + "D101", # don't require docs for every class + "D102", # don't require docs for every class method + "D103", # don't require docs for every function + "D104", # don't require docs for every package + "D106", # don't require docs for every nested class + "D107", # don't require docs for __init__ + "D417", # don't require docs for every function parameter +] + +[tool.ruff.lint.isort] +force-single-line = true + +[tool.ruff.lint.pydocstyle] +# enable the google doc style rules by default +convention = "google" + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["S101"] # assert is the point in tests diff --git a/python/understack-neutron-flavors/tests/__init__.py b/python/understack-neutron-flavors/tests/__init__.py new file mode 100644 index 000000000..1eb5ee423 --- /dev/null +++ b/python/understack-neutron-flavors/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for understack-neutron-flavors.""" diff --git a/python/understack-neutron-flavors/tests/test_router_flavors.py b/python/understack-neutron-flavors/tests/test_router_flavors.py new file mode 100644 index 000000000..26b92dfc3 --- /dev/null +++ b/python/understack-neutron-flavors/tests/test_router_flavors.py @@ -0,0 +1,451 @@ +import json +import sys +import unittest +from pathlib import Path + +from understack_neutron_flavors import create_router_flavors +from understack_neutron_flavors import delete_router_flavors +from understack_neutron_flavors import router_flavors +from understack_neutron_flavors import router_flavors_common as common +from understack_neutron_flavors import update_router_flavors + +PROJECT_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_DIR)) + + +class NotFound(Exception): + status_code = 404 + + +class Conflict(Exception): + status_code = 409 + + +class FakeNetwork: + def __init__(self, flavors, profiles, routers=None): + self._flavors = {flavor["id"]: dict(flavor) for flavor in flavors} + self._profiles = {profile["id"]: dict(profile) for profile in profiles} + self._routers = [dict(router) for router in routers or []] + self.deleted_flavors = [] + self.deleted_profiles = [] + self.created_flavors = [] + self.created_profiles = [] + self.updated_flavors = [] + + def flavors(self, **query): + flavors = list(self._flavors.values()) + if "service_type" in query: + flavors = [ + flavor + for flavor in flavors + if flavor.get("service_type") == query["service_type"] + ] + if "name" in query: + flavors = [ + flavor for flavor in flavors if flavor.get("name") == query["name"] + ] + return iter(flavors) + + def get_service_profile(self, profile_id): + try: + return self._profiles[profile_id] + except KeyError as exc: + raise NotFound(profile_id) from exc + + def service_profiles(self): + return iter(self._profiles.values()) + + def create_service_profile(self, **attrs): + profile_id = f"sp-created-{len(self._profiles) + 1}" + profile = {"id": profile_id, **attrs} + self._profiles[profile_id] = profile + self.created_profiles.append(profile) + return profile + + def get_flavor(self, flavor): + flavor_id = flavor["id"] if isinstance(flavor, dict) else flavor + try: + return self._flavors[flavor_id] + except KeyError as exc: + raise NotFound(flavor_id) from exc + + def create_flavor(self, **attrs): + flavor_id = f"fl-created-{len(self._flavors) + 1}" + flavor = {"id": flavor_id, **attrs} + self._flavors[flavor_id] = flavor + self.created_flavors.append(flavor) + return flavor + + def update_flavor(self, flavor, **attrs): + flavor_id = flavor["id"] if isinstance(flavor, dict) else flavor + current = self.get_flavor(flavor_id) + current.update(attrs) + self.updated_flavors.append({"id": flavor_id, **attrs}) + return current + + def delete_flavor(self, flavor, ignore_missing=True): + flavor_id = flavor["id"] if isinstance(flavor, dict) else flavor + if flavor_id not in self._flavors: + if ignore_missing: + return + raise NotFound(flavor_id) + del self._flavors[flavor_id] + self.deleted_flavors.append(flavor_id) + + def delete_service_profile(self, profile, ignore_missing=True): + profile_id = profile["id"] if isinstance(profile, dict) else profile + if profile_id not in self._profiles: + if ignore_missing: + return + raise NotFound(profile_id) + for flavor in self._flavors.values(): + if profile_id in common.service_profile_ids(flavor): + raise Conflict(profile_id) + del self._profiles[profile_id] + self.deleted_profiles.append(profile_id) + + def routers(self, **query): + flavor_id = query.get("flavor_id") + routers = self._routers + if flavor_id: + routers = [ + router for router in routers if router.get("flavor_id") == flavor_id + ] + return iter(routers) + + +class FakeConnection: + def __init__(self, network): + self.network = network + + +class HookConfigTest(unittest.TestCase): + def test_hook_config_has_hourly_schedule(self): + self.assertEqual( + [ + { + "name": "hourly sync", + "crontab": common.SYNC_CRONTAB, + } + ], + router_flavors.HOOK_CONFIG["schedule"], + ) + + +class UpdateFlavorTest(unittest.TestCase): + def test_ensure_flavor_marks_created_flavor_description(self): + conn = FakeConnection(FakeNetwork(flavors=[], profiles=[])) + + flavor = update_router_flavors.ensure_flavor( + conn, + "pa1410", + "L3_ROUTER_NAT", + "Physical PA 1410", + ) + + self.assertEqual( + common.managed_flavor_description("Physical PA 1410"), + flavor["description"], + ) + + def test_ensure_flavor_marks_existing_flavor_description(self): + conn = FakeConnection( + FakeNetwork( + flavors=[ + { + "id": "fl-existing", + "name": "pa1410", + "service_type": "L3_ROUTER_NAT", + "description": "Physical PA 1410", + }, + ], + profiles=[], + ) + ) + + flavor = update_router_flavors.ensure_flavor( + conn, + "pa1410", + "L3_ROUTER_NAT", + "Physical PA 1410", + ) + + self.assertEqual( + common.managed_flavor_description("Physical PA 1410"), + flavor["description"], + ) + self.assertEqual( + [ + { + "id": "fl-existing", + "description": common.managed_flavor_description( + "Physical PA 1410" + ), + } + ], + conn.network.updated_flavors, + ) + + +class PruneRemovedFlavorsTest(unittest.TestCase): + def setUp(self): + self._old_prune = common.PRUNE_REMOVED_FLAVORS + self._old_delete_profiles = common.DELETE_UNUSED_SERVICE_PROFILES + self._old_prefixes = common.PRUNE_DRIVER_PREFIXES + + common.PRUNE_REMOVED_FLAVORS = True + common.DELETE_UNUSED_SERVICE_PROFILES = True + common.PRUNE_DRIVER_PREFIXES = ("neutron_understack.l3_router.",) + + def tearDown(self): + common.PRUNE_REMOVED_FLAVORS = self._old_prune + common.DELETE_UNUSED_SERVICE_PROFILES = self._old_delete_profiles + common.PRUNE_DRIVER_PREFIXES = self._old_prefixes + + def test_prune_deletes_removed_flavor_and_orphan_profile(self): + conn = FakeConnection( + FakeNetwork( + flavors=[ + { + "id": "fl-keep", + "name": "keep", + "service_type": "L3_ROUTER_NAT", + "service_profile_ids": ["sp-keep"], + }, + { + "id": "fl-remove", + "name": "remove", + "service_type": "L3_ROUTER_NAT", + "description": common.managed_flavor_description("remove"), + "service_profile_ids": ["sp-remove"], + }, + ], + profiles=[ + { + "id": "sp-keep", + "driver": "neutron_understack.l3_router.vrf.Vrf", + }, + { + "id": "sp-remove", + "driver": "neutron_understack.l3_router.cisco_asa.CiscoAsa", + "meta_info": common.OPERATOR_META_INFO_MARKERS, + }, + ], + ) + ) + + delete_router_flavors.prune_removed_flavors(conn, [{"name": "keep"}]) + + self.assertEqual(["fl-remove"], conn.network.deleted_flavors) + self.assertEqual(["sp-remove"], conn.network.deleted_profiles) + + def test_prune_deletes_marked_flavor_and_keeps_unmanaged_profile(self): + conn = FakeConnection( + FakeNetwork( + flavors=[ + { + "id": "fl-remove", + "name": "remove", + "service_type": "L3_ROUTER_NAT", + "description": common.managed_flavor_description("remove"), + "service_profile_ids": ["sp-external"], + }, + ], + profiles=[ + { + "id": "sp-external", + "driver": "neutron_understack.l3_router.cisco_asa.CiscoAsa", + "meta_info": {}, + }, + ], + ) + ) + + delete_router_flavors.prune_removed_flavors(conn, []) + + self.assertEqual(["fl-remove"], conn.network.deleted_flavors) + self.assertEqual([], conn.network.deleted_profiles) + + def test_prune_skips_unmarked_flavor_with_unmanaged_profile(self): + conn = FakeConnection( + FakeNetwork( + flavors=[ + { + "id": "fl-manual", + "name": "manual", + "service_type": "L3_ROUTER_NAT", + "service_profile_ids": ["sp-manual"], + }, + ], + profiles=[ + { + "id": "sp-manual", + "driver": "neutron_understack.l3_router.cisco_asa.CiscoAsa", + "meta_info": {}, + }, + ], + ) + ) + + delete_router_flavors.prune_removed_flavors(conn, []) + + self.assertEqual([], conn.network.deleted_flavors) + self.assertEqual([], conn.network.deleted_profiles) + + def test_prune_keeps_profile_id_configured_in_current_data(self): + conn = FakeConnection( + FakeNetwork( + flavors=[ + { + "id": "fl-remove", + "name": "remove", + "service_type": "L3_ROUTER_NAT", + "description": common.managed_flavor_description("remove"), + "service_profile_ids": ["sp-protected"], + }, + ], + profiles=[ + { + "id": "sp-protected", + "driver": "neutron_understack.l3_router.cisco_asa.CiscoAsa", + }, + ], + ) + ) + + delete_router_flavors.prune_removed_flavors( + conn, + [{"name": "keep", "profile_id": "sp-protected"}], + ) + + self.assertEqual(["fl-remove"], conn.network.deleted_flavors) + self.assertEqual([], conn.network.deleted_profiles) + + def test_prune_keeps_profile_still_attached_to_another_flavor(self): + conn = FakeConnection( + FakeNetwork( + flavors=[ + { + "id": "fl-keep", + "name": "keep", + "service_type": "L3_ROUTER_NAT", + "service_profile_ids": ["sp-shared"], + }, + { + "id": "fl-remove", + "name": "remove", + "service_type": "L3_ROUTER_NAT", + "service_profile_ids": ["sp-shared"], + }, + ], + profiles=[ + { + "id": "sp-shared", + "driver": "neutron_understack.l3_router.vrf.Vrf", + "meta_info": common.OPERATOR_META_INFO_MARKERS, + }, + ], + ) + ) + + delete_router_flavors.prune_removed_flavors(conn, [{"name": "keep"}]) + + self.assertEqual(["fl-remove"], conn.network.deleted_flavors) + self.assertEqual([], conn.network.deleted_profiles) + + def test_prune_skips_removed_flavor_still_used_by_router(self): + conn = FakeConnection( + FakeNetwork( + flavors=[ + { + "id": "fl-remove", + "name": "remove", + "service_type": "L3_ROUTER_NAT", + "service_profile_ids": ["sp-remove"], + }, + ], + profiles=[ + { + "id": "sp-remove", + "driver": "neutron_understack.l3_router.cisco_asa.CiscoAsa", + }, + ], + routers=[{"id": "router-1", "flavor_id": "fl-remove"}], + ) + ) + + delete_router_flavors.prune_removed_flavors(conn, []) + + self.assertEqual([], conn.network.deleted_flavors) + self.assertEqual([], conn.network.deleted_profiles) + + def test_ensure_profile_marks_created_profile_without_configured_id(self): + conn = FakeConnection(FakeNetwork(flavors=[], profiles=[])) + + profile = create_router_flavors.ensure_profile( + conn, + "pa1410", + "neutron_understack.l3_router.palo_alto.PaloAlto", + "Physical PA 1410", + {"resource_class": "pa1410"}, + "", + ) + + meta_info = json.loads(profile["meta_info"]) + self.assertEqual("pa1410", meta_info["resource_class"]) + for key, value in common.OPERATOR_META_INFO_MARKERS.items(): + self.assertEqual(value, meta_info[key]) + + def test_matching_profile_ignores_managed_marker(self): + conn = FakeConnection( + FakeNetwork( + flavors=[], + profiles=[ + { + "id": "sp-managed", + "driver": "neutron_understack.l3_router.vrf.Vrf", + "meta_info": { + "vni_alloc": "auto", + **common.OPERATOR_META_INFO_MARKERS, + }, + }, + ], + ) + ) + + profile = create_router_flavors.find_matching_profile( + conn, + "neutron_understack.l3_router.vrf.Vrf", + {"vni_alloc": "auto"}, + ) + + self.assertEqual("sp-managed", profile["id"]) + + def test_prune_ignores_profiles_outside_driver_scope(self): + conn = FakeConnection( + FakeNetwork( + flavors=[ + { + "id": "fl-external", + "name": "external", + "service_type": "L3_ROUTER_NAT", + "service_profile_ids": ["sp-external"], + }, + ], + profiles=[ + { + "id": "sp-external", + "driver": "third.party.Router", + }, + ], + ) + ) + + delete_router_flavors.prune_removed_flavors(conn, []) + + self.assertEqual([], conn.network.deleted_flavors) + self.assertEqual([], conn.network.deleted_profiles) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/understack-neutron-flavors/understack_neutron_flavors/__init__.py b/python/understack-neutron-flavors/understack_neutron_flavors/__init__.py new file mode 100644 index 000000000..baf6f61fe --- /dev/null +++ b/python/understack-neutron-flavors/understack_neutron_flavors/__init__.py @@ -0,0 +1 @@ +"""Neutron shell-operator helpers.""" diff --git a/python/understack-neutron-flavors/understack_neutron_flavors/create_router_flavors.py b/python/understack-neutron-flavors/understack_neutron_flavors/create_router_flavors.py new file mode 100644 index 000000000..736df2bbe --- /dev/null +++ b/python/understack-neutron-flavors/understack_neutron_flavors/create_router_flavors.py @@ -0,0 +1,113 @@ +"""Create helpers for Neutron router flavors and service profiles.""" + +from __future__ import annotations + +from typing import Any + +from . import router_flavors_common as common + + +def find_matching_profile(conn: Any, driver: str, meta_info: Any) -> Any | None: + matching_profiles = [] + for profile in conn.network.service_profiles(): + if common.get_value(profile, "driver", "Driver", default="") != driver: + continue + + if common.meta_info_matches( + common.service_profile_meta_info(profile), + meta_info, + ): + matching_profiles.append(profile) + + for profile in matching_profiles: + if common.is_managed_service_profile(profile): + return profile + + return matching_profiles[0] if matching_profiles else None + + +def ensure_profile( + conn: Any, + name: str, + driver: str, + description: str, + meta_info: Any, + configured_profile_id: str, +) -> Any: + if configured_profile_id: + profile = common.get_service_profile(conn, configured_profile_id) + if profile: + common.log( + f"Using configured service profile {configured_profile_id} for {name}" + ) + return profile + + common.log( + f"Configured service profile {configured_profile_id} " + f"for {name} was not found" + ) + + profile = find_matching_profile(conn, driver, meta_info) + if profile: + profile_id = common.resource_id(profile) + common.log(f"Reusing service profile {profile_id} for {name}") + # Neutron rejects service profile updates once they are used by + # service instances. Matching driver/meta_info is enough for + # idempotent reuse. + return profile + + service_profile_meta = meta_info + if not configured_profile_id: + service_profile_meta = common.managed_meta_info(meta_info) + + common.log(f"Creating service profile for {name} driver={driver}") + return conn.network.create_service_profile( + description=description, + driver=driver, + meta_info=common.meta_info_payload(service_profile_meta), + is_enabled=True, + ) + + +def find_flavor(conn: Any, name: str) -> Any | None: + for flavor in conn.network.flavors(name=name): + if common.get_value(flavor, "name", "Name") == name: + return flavor + + return None + + +def create_flavor(conn: Any, name: str, service_type: str, description: str) -> Any: + common.log(f"Creating router flavor {name} service_type={service_type}") + attrs = { + "name": name, + "service_type": service_type, + "is_enabled": True, + "description": common.managed_flavor_description(description), + } + return conn.network.create_flavor(**attrs) + + +def ensure_profile_attached(conn: Any, flavor: Any, profile: Any) -> Any: + flavor = conn.network.get_flavor(flavor) + flavor_id = common.resource_id(flavor) + profile_id = common.resource_id(profile) + + if profile_id in common.service_profile_ids(flavor): + flavor_name = common.get_value(flavor, "name", "Name", default=flavor_id) + common.log( + f"Router flavor {flavor_name} already has service profile {profile_id}" + ) + return flavor + + common.log(f"Binding service profile {profile_id} to router flavor {flavor_id}") + try: + conn.network.associate_flavor_with_service_profile(flavor, profile) + except Exception as exc: + if not common.is_conflict(exc): + raise + common.log( + f"Router flavor {flavor_id} already has service profile {profile_id}" + ) + + return conn.network.get_flavor(flavor) diff --git a/python/understack-neutron-flavors/understack_neutron_flavors/delete_router_flavors.py b/python/understack-neutron-flavors/understack_neutron_flavors/delete_router_flavors.py new file mode 100644 index 000000000..e1a1ab42b --- /dev/null +++ b/python/understack-neutron-flavors/understack_neutron_flavors/delete_router_flavors.py @@ -0,0 +1,209 @@ +"""Delete/prune logic for removed Neutron router flavors.""" + +from __future__ import annotations + +from typing import Any + +from . import router_flavors_common as common + + +def configured_service_profile_ids(flavors: list[dict[str, Any]]) -> set[str]: + return { + str(flavor_config["profile_id"]) + for flavor_config in flavors + if flavor_config.get("profile_id") + } + + +def configured_flavor_names(flavors: list[dict[str, Any]]) -> set[str]: + return { + str(flavor_config["name"]) + for flavor_config in flavors + if flavor_config.get("name") + } + + +def service_profile_driver(profile: Any) -> str: + return str(common.get_value(profile, "driver", "Driver", default="")) + + +def get_cached_service_profile( + conn: Any, + profile_id: str, + profile_cache: dict[str, Any | None], +) -> Any | None: + if profile_id not in profile_cache: + profile_cache[profile_id] = common.get_service_profile(conn, profile_id) + return profile_cache[profile_id] + + +def is_prunable_service_profile(profile: Any) -> bool: + driver = service_profile_driver(profile) + return bool(common.PRUNE_DRIVER_PREFIXES) and any( + driver.startswith(prefix) for prefix in common.PRUNE_DRIVER_PREFIXES + ) + + +def is_prunable_flavor( + conn: Any, + flavor: Any, + profile_cache: dict[str, Any | None], +) -> bool: + if ( + common.get_value(flavor, "service_type", "Service Type") + != common.DEFAULT_SERVICE_TYPE + ): + return False + + if common.is_managed_flavor(flavor): + return True + + for profile_id in common.service_profile_ids(flavor): + profile = get_cached_service_profile(conn, profile_id, profile_cache) + if ( + profile + and common.is_managed_service_profile(profile) + and is_prunable_service_profile(profile) + ): + return True + + return False + + +def flavor_has_routers(conn: Any, flavor: Any) -> bool: + flavor_id = common.resource_id(flavor) + flavor_name = common.get_value(flavor, "name", "Name", default=flavor_id) + + try: + routers = list(conn.network.routers(flavor_id=flavor_id)) + except Exception as exc: + common.log( + f"Unable to check routers for removed router flavor {flavor_name}; " + f"skipping deletion: {exc}" + ) + return True + + if routers: + common.log( + f"Router flavor {flavor_name} is still used by {len(routers)} " + "router(s); skipping deletion" + ) + return True + + return False + + +def service_profile_attached_to_any_flavor(conn: Any, profile_id: str) -> bool: + for flavor in conn.network.flavors(service_type=common.DEFAULT_SERVICE_TYPE): + if profile_id in common.service_profile_ids(flavor): + return True + return False + + +def maybe_delete_service_profile( + conn: Any, + profile_id: str, + protected_profile_ids: set[str], + profile_cache: dict[str, Any | None], +) -> None: + if not common.DELETE_UNUSED_SERVICE_PROFILES: + common.log(f"Keeping service profile {profile_id}; profile pruning is disabled") + return + + if profile_id in protected_profile_ids: + common.log( + f"Keeping service profile {profile_id}; it is configured by " + "router_flavors.json" + ) + return + + profile = get_cached_service_profile(conn, profile_id, profile_cache) + if not profile: + return + + if not is_prunable_service_profile(profile): + common.log( + f"Keeping service profile {profile_id}; driver " + f"{service_profile_driver(profile)} is outside prune scope" + ) + return + + if not common.is_managed_service_profile(profile): + common.log(f"Keeping service profile {profile_id}; it is not operator-managed") + return + + if service_profile_attached_to_any_flavor(conn, profile_id): + common.log(f"Keeping service profile {profile_id}; it is still attached") + return + + common.log(f"Deleting unused service profile {profile_id}") + try: + conn.network.delete_service_profile(profile, ignore_missing=True) + profile_cache[profile_id] = None + except Exception as exc: + if common.is_not_found(exc): + profile_cache[profile_id] = None + return + if common.is_conflict(exc): + common.log(f"Service profile {profile_id} is still in use; skipping delete") + return + raise + + +def delete_removed_flavor( + conn: Any, + flavor: Any, + protected_profile_ids: set[str], + profile_cache: dict[str, Any | None], +) -> None: + flavor_id = common.resource_id(flavor) + flavor_name = common.get_value(flavor, "name", "Name", default=flavor_id) + profile_ids = common.service_profile_ids(flavor) + + if flavor_has_routers(conn, flavor): + return + + common.log(f"Deleting removed router flavor {flavor_name} ({flavor_id})") + try: + conn.network.delete_flavor(flavor, ignore_missing=True) + except Exception as exc: + if common.is_not_found(exc): + return + if common.is_conflict(exc): + common.log(f"Router flavor {flavor_name} is still in use; skipping delete") + return + raise + + for profile_id in profile_ids: + maybe_delete_service_profile( + conn, + profile_id, + protected_profile_ids, + profile_cache, + ) + + +def prune_removed_flavors(conn: Any, flavors: list[dict[str, Any]]) -> None: + if not common.PRUNE_REMOVED_FLAVORS: + common.log("Router flavor pruning is disabled") + return + + desired_names = configured_flavor_names(flavors) + protected_profile_ids = configured_service_profile_ids(flavors) + profile_cache: dict[str, Any | None] = {} + + common.log("Pruning removed router flavors") + for flavor in list(conn.network.flavors(service_type=common.DEFAULT_SERVICE_TYPE)): + flavor_name = common.get_value(flavor, "name", "Name") + if not flavor_name or flavor_name in desired_names: + continue + + if not is_prunable_flavor(conn, flavor, profile_cache): + continue + + delete_removed_flavor( + conn, + flavor, + protected_profile_ids, + profile_cache, + ) diff --git a/python/understack-neutron-flavors/understack_neutron_flavors/router_flavors.py b/python/understack-neutron-flavors/understack_neutron_flavors/router_flavors.py new file mode 100644 index 000000000..18cae5ab9 --- /dev/null +++ b/python/understack-neutron-flavors/understack_neutron_flavors/router_flavors.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Reconcile Neutron router flavors and service profiles from JSON config.""" + +from __future__ import annotations + +import json +import os +import sys + +from understack_neutron_flavors import router_flavors_common as common +from understack_neutron_flavors.delete_router_flavors import prune_removed_flavors +from understack_neutron_flavors.update_router_flavors import sync_flavor + +HOOK_CONFIG = { + "configVersion": "v1", + "onStartup": 1, + "schedule": [ + { + "name": "hourly sync", + "crontab": common.SYNC_CRONTAB, + } + ], + "settings": { + "executionMinInterval": "30s", + "executionBurst": 1, + }, +} + + +def run() -> int: + if len(sys.argv) > 1 and sys.argv[1] == "--config": + print(json.dumps(HOOK_CONFIG, indent=2)) + return 0 + + flavors = common.load_config(common.CONFIG_PATH) + conn = common.connect_openstack(os.environ.get("OS_CLOUD")) + common.wait_for_openstack_network(conn) + + common.log(f"Found {len(flavors)} router flavor(s) to reconcile") + for flavor_config in flavors: + sync_flavor(conn, flavor_config) + + prune_removed_flavors(conn, flavors) + + common.log("Finished reconciling router flavors") + return 0 + + +def main() -> None: + try: + sys.exit(run()) + except Exception as exc: + common.log(str(exc)) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/python/understack-neutron-flavors/understack_neutron_flavors/router_flavors_common.py b/python/understack-neutron-flavors/understack_neutron_flavors/router_flavors_common.py new file mode 100644 index 000000000..adb47dd08 --- /dev/null +++ b/python/understack-neutron-flavors/understack_neutron_flavors/router_flavors_common.py @@ -0,0 +1,309 @@ +"""Shared helpers for Neutron router flavor reconciliation.""" + +from __future__ import annotations + +import ast +import json +import os +import sys +import time +from typing import Any + + +def env_bool(name: str, default: bool) -> bool: + value = os.environ.get(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def env_tuple(name: str, default: str) -> tuple[str, ...]: + return tuple( + item.strip() + for item in os.environ.get(name, default).split(",") + if item.strip() + ) + + +SYNC_CRONTAB = os.environ.get("NEUTRON_ROUTER_FLAVOR_SYNC_CRONTAB", "0 * * * *") +CONFIG_PATH = os.environ.get( + "NEUTRON_ROUTER_FLAVORS_CONFIG", + "/etc/neutron-router-flavors/router_flavors.json", +) +DEFAULT_SERVICE_TYPE = os.environ.get( + "NEUTRON_ROUTER_FLAVOR_SERVICE_TYPE", + "L3_ROUTER_NAT", +) +PRUNE_REMOVED_FLAVORS = env_bool("NEUTRON_ROUTER_FLAVOR_PRUNE", False) +DELETE_UNUSED_SERVICE_PROFILES = env_bool( + "NEUTRON_ROUTER_FLAVOR_DELETE_UNUSED_PROFILES", + True, +) +PRUNE_DRIVER_PREFIXES = env_tuple( + "NEUTRON_ROUTER_FLAVOR_PRUNE_DRIVER_PREFIXES", + "neutron_understack.l3_router.", +) +MANAGED_META_INFO_KEY = os.environ.get( + "NEUTRON_ROUTER_FLAVOR_MANAGED_META_INFO_KEY", + "_understack_router_flavor_operator", +) +MANAGED_META_INFO_VALUE = "managed" +FLAVOR_DESCRIPTION_MARKER = os.environ.get( + "NEUTRON_ROUTER_FLAVOR_DESCRIPTION_MARKER", + "[understack-router-flavor-operator]", +) +MARKER_VERSION_META_INFO_KEY = "_understack_router_flavor_marker_version" +MARKER_VERSION_META_INFO_VALUE = "v1" +MARKER_SOURCE_META_INFO_KEY = "_understack_router_flavor_source" +MARKER_SOURCE_META_INFO_VALUE = os.path.basename(CONFIG_PATH) or "router_flavors.json" +OPERATOR_META_INFO_MARKERS = { + MANAGED_META_INFO_KEY: MANAGED_META_INFO_VALUE, + MARKER_VERSION_META_INFO_KEY: MARKER_VERSION_META_INFO_VALUE, + MARKER_SOURCE_META_INFO_KEY: MARKER_SOURCE_META_INFO_VALUE, +} +OPERATOR_META_INFO_KEYS = frozenset(OPERATOR_META_INFO_MARKERS) +READY_RETRIES = int(os.environ.get("NEUTRON_ROUTER_FLAVOR_READY_RETRIES", "30")) +READY_DELAY = float(os.environ.get("NEUTRON_ROUTER_FLAVOR_READY_DELAY", "10")) +_MISSING = object() + +# Markers used by this hook: +# - flavor.description contains [understack-router-flavor-operator]: ownership +# marker for router flavors. This lets the hook distinguish config-managed +# flavors from manually created flavors, including flavors that use an +# externally configured profile_id. +# - service_profile.meta_info["_understack_router_flavor_operator"]="managed": +# ownership marker for service profiles. Destructive profile cleanup requires +# this exact marker so manual service profiles are not deleted. +# - service_profile.meta_info["_understack_router_flavor_marker_version"]="v1": +# marker schema version for future migrations. +# - service_profile.meta_info["_understack_router_flavor_source"]=: +# traceability marker showing where the service profile was sourced from. +# +# Router flavors do not expose service-profile-style meta_info in the API used +# here, so the flavor marker is stored in description. Keep it compact because +# users may see the description in OpenStack output. + + +class ConfigError(Exception): + pass + + +def log(message: str) -> None: + print(f"[router_flavors] {message}", file=sys.stderr) + + +def _resource_value(resource: Any, name: str) -> Any: + if isinstance(resource, dict): + return resource[name] if name in resource else _MISSING + + getter = getattr(resource, "get", None) + if callable(getter): + try: + value = getter(name, _MISSING) + except TypeError: + try: + value = getter(name) + except Exception: + value = _MISSING + except Exception: + value = _MISSING + + if value is not _MISSING: + return value + + value = getattr(resource, name, _MISSING) + if value is not _MISSING: + return value + + try: + data = resource.to_dict(computed=False) + except Exception: + data = {} + + return data[name] if name in data else _MISSING + + +def get_value(resource: Any, *names: str, default: Any = None) -> Any: + for name in names: + value = _resource_value(resource, name) + if value is not _MISSING and value is not None: + return value + + return default + + +def resource_id(resource: Any) -> str: + value = get_value(resource, "id", "ID", "Id") + if not value: + raise RuntimeError(f"Unable to read ID from resource {resource!r}") + return str(value) + + +def normalize_meta_info(value: Any) -> Any: + if value is None or value == "": + return {} + + if isinstance(value, str): + text = value.strip() + if not text: + return {} + + try: + return json.loads(text) + except json.JSONDecodeError: + try: + return ast.literal_eval(text) + except (SyntaxError, ValueError): + return text + + return value + + +def meta_info_payload(value: Any) -> str: + normalized = normalize_meta_info(value) + return json.dumps(normalized, sort_keys=True, separators=(",", ":")) + + +def comparable_meta_info(value: Any) -> Any: + normalized = normalize_meta_info(value) + if isinstance(normalized, dict): + return { + key: item + for key, item in normalized.items() + if key not in OPERATOR_META_INFO_KEYS + } + return normalized + + +def meta_info_matches(current: Any, desired: Any) -> bool: + return meta_info_payload(comparable_meta_info(current)) == meta_info_payload( + comparable_meta_info(desired) + ) + + +def managed_meta_info(value: Any) -> Any: + normalized = normalize_meta_info(value) + if not isinstance(normalized, dict): + return normalized + + managed = dict(normalized) + managed.update(OPERATOR_META_INFO_MARKERS) + return managed + + +def clean_flavor_description(value: Any) -> str: + description = "" if value is None else str(value) + return description.replace(FLAVOR_DESCRIPTION_MARKER, "").strip() + + +def managed_flavor_description(value: Any) -> str: + description = clean_flavor_description(value) + if not description: + return FLAVOR_DESCRIPTION_MARKER + return f"{description} {FLAVOR_DESCRIPTION_MARKER}" + + +def flavor_description_has_marker(value: Any) -> bool: + return FLAVOR_DESCRIPTION_MARKER in str(value or "") + + +def is_managed_flavor(flavor: Any) -> bool: + return flavor_description_has_marker( + get_value(flavor, "description", "Description", default="") + ) + + +def service_profile_meta_info(profile: Any) -> Any: + return get_value(profile, "meta_info", "metainfo", default={}) + + +def is_managed_service_profile(profile: Any) -> bool: + meta_info = normalize_meta_info(service_profile_meta_info(profile)) + return ( + isinstance(meta_info, dict) + and meta_info.get(MANAGED_META_INFO_KEY) == MANAGED_META_INFO_VALUE + ) + + +def is_not_found(exc: Exception) -> bool: + return getattr(exc, "status_code", None) == 404 or exc.__class__.__name__ in { + "NotFoundException", + "ResourceNotFound", + } + + +def is_conflict(exc: Exception) -> bool: + return ( + getattr(exc, "status_code", None) == 409 + or exc.__class__.__name__ in {"ConflictException", "ResourceConflict"} + or "already" in str(exc).lower() + ) + + +def connect_openstack(os_cloud: str | None) -> Any: + try: + import openstack + except ImportError as exc: + raise RuntimeError("openstacksdk is required to run this hook") from exc + + return openstack.connect(cloud=os_cloud) + + +def load_config(path: str) -> list[dict[str, Any]]: + if not os.path.isfile(path): + raise ConfigError(f"Router flavor config not found at {path}") + + with open(path, encoding="utf-8") as config_file: + flavors = json.load(config_file) + + if not isinstance(flavors, list): + raise ConfigError("Router flavor config must be a JSON list") + + return flavors + + +def wait_for_openstack_network(conn: Any) -> None: + for attempt in range(1, READY_RETRIES + 1): + try: + next(iter(conn.network.flavors()), None) + return + except Exception as exc: + if attempt >= READY_RETRIES: + raise RuntimeError( + f"Neutron API did not become ready after {READY_RETRIES} attempt(s)" + ) from exc + + log(f"Waiting for Neutron API ({attempt}/{READY_RETRIES}): {exc}") + time.sleep(READY_DELAY) + + +def get_service_profile(conn: Any, profile_id: str) -> Any | None: + try: + return conn.network.get_service_profile(profile_id) + except Exception as exc: + if is_not_found(exc): + return None + raise + + +def service_profile_ids(flavor: Any) -> list[str]: + profiles = get_value( + flavor, + "service_profile_ids", + "service_profiles", + "profiles", + default=[], + ) + if profiles is None: + return [] + if isinstance(profiles, str): + return [item.strip() for item in profiles.split(",") if item.strip()] + return [str(profile) for profile in profiles] + + +def config_meta_info(flavor_config: dict[str, Any]) -> Any: + if "metainfo" in flavor_config: + name = flavor_config.get("name", "") + raise ConfigError(f"Router flavor {name} uses metainfo; use meta_info instead") + + return flavor_config.get("meta_info", {}) diff --git a/python/understack-neutron-flavors/understack_neutron_flavors/update_router_flavors.py b/python/understack-neutron-flavors/understack_neutron_flavors/update_router_flavors.py new file mode 100644 index 000000000..168a3175a --- /dev/null +++ b/python/understack-neutron-flavors/understack_neutron_flavors/update_router_flavors.py @@ -0,0 +1,69 @@ +"""Update and sync logic for configured Neutron router flavors.""" + +from __future__ import annotations + +import json +from typing import Any + +from . import create_router_flavors +from . import router_flavors_common as common + + +def ensure_flavor(conn: Any, name: str, service_type: str, description: str) -> Any: + flavor = create_router_flavors.find_flavor(conn, name) + managed_description = common.managed_flavor_description(description) + if flavor: + common.log(f"Router flavor {name} already exists") + current_description = common.get_value( + flavor, + "description", + "Description", + default="", + ) + description_changed = common.clean_flavor_description( + current_description + ) != common.clean_flavor_description(description) + marker_missing = not common.flavor_description_has_marker(current_description) + if description_changed or marker_missing: + return conn.network.update_flavor(flavor, description=managed_description) + return flavor + + return create_router_flavors.create_flavor(conn, name, service_type, description) + + +def render_flavor(flavor: Any) -> dict[str, Any]: + return { + "id": common.get_value(flavor, "id", "ID"), + "name": common.get_value(flavor, "name", "Name"), + "service_type": common.get_value(flavor, "service_type", "Service Type"), + "description": common.get_value(flavor, "description", "Description"), + "service_profile_ids": common.service_profile_ids(flavor), + } + + +def sync_flavor(conn: Any, flavor_config: dict[str, Any]) -> None: + name = flavor_config.get("name") + driver = flavor_config.get("driver") + if not name or not driver: + raise common.ConfigError( + "Each router flavor entry must define name and driver: " f"{flavor_config}" + ) + + description = flavor_config.get("description", "") + profile_description = flavor_config.get("profile_description", description) + service_type = flavor_config.get("service_type", common.DEFAULT_SERVICE_TYPE) + profile_id = flavor_config.get("profile_id", "") + meta_info = common.config_meta_info(flavor_config) + + common.log(f"Reconciling router flavor {name}") + profile = create_router_flavors.ensure_profile( + conn, + name, + driver, + profile_description, + meta_info, + profile_id, + ) + flavor = ensure_flavor(conn, name, service_type, description) + flavor = create_router_flavors.ensure_profile_attached(conn, flavor, profile) + print(json.dumps(render_flavor(flavor), sort_keys=True))