From 68070419bd27f9d0ac424032a6b03b86aea1125a Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Thu, 2 Jul 2026 12:46:16 +0200 Subject: [PATCH 1/5] Add SSE events to API v2 Detail/list endpoints gain a parameter to toggle streaming. With this set, SSE is used to return the initial representation once and watch the database for future changes. --- requirements.txt | 1 + simplyblock_core/db_controller.py | 5 +- simplyblock_core/models/base_model.py | 36 +- simplyblock_core/models/cluster.py | 2 + simplyblock_core/models/job_schedule.py | 2 + simplyblock_core/models/lvol_model.py | 2 + simplyblock_core/models/pool.py | 2 + simplyblock_core/models/snapshot.py | 2 + simplyblock_core/models/storage_node.py | 2 + simplyblock_core/watches.py | 33 ++ simplyblock_web/api/v2/_watch.py | 358 ++++++++++++++++++ simplyblock_web/api/v2/cluster/__init__.py | 29 +- .../api/v2/cluster/storage_node/__init__.py | 37 +- .../api/v2/cluster/storage_node/device.py | 55 ++- .../api/v2/cluster/storage_pool/__init__.py | 34 +- .../api/v2/cluster/storage_pool/snapshot.py | 38 +- .../cluster/storage_pool/volume/__init__.py | 44 ++- simplyblock_web/api/v2/cluster/task.py | 37 +- tests/unit/conftest.py | 8 +- tests/unit/test_watch_counters.py | 106 ++++++ tests/unit/web/api/v2/test_watch_stream.py | 308 +++++++++++++++ 21 files changed, 1100 insertions(+), 41 deletions(-) create mode 100644 simplyblock_core/watches.py create mode 100644 simplyblock_web/api/v2/_watch.py create mode 100644 tests/unit/test_watch_counters.py create mode 100644 tests/unit/web/api/v2/test_watch_stream.py diff --git a/requirements.txt b/requirements.txt index 6297959f3..b00a1f75d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,7 @@ sentry-sdk[flask]>=1.40 flask-openapi3 jsonschema fastapi +sse-starlette uvicorn prometheus_api_client paramiko diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index cf522e1d0..62b05d0f2 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -7,7 +7,7 @@ import fdb from typing import Any, List, Optional -from simplyblock_core import constants +from simplyblock_core import constants, watches from simplyblock_core.models.cluster import Cluster, ClusterAddNodeLock, PortReservation from simplyblock_core.models.events import EventObj from simplyblock_core.models.job_schedule import JobSchedule @@ -675,6 +675,8 @@ def _atomic_update_tx(self, tr, key, model_cls, mutate_fn): if mutate_fn(obj) is False: return obj tr[key] = json.dumps(obj.to_dict(unwrap_secrets=True)).encode() + if getattr(model_cls, '_WATCHED', False): + tr.add(watches.watch_counter_key(model_cls), watches.ONE_LE64) return obj def atomic_update(self, obj, mutate_fn): @@ -900,6 +902,7 @@ def _try_set_node_restarting_tx(self, tr, cluster_id, node_id): prefix = target.get_db_id() data = json.dumps(target.get_clean_dict(unwrap_secrets=True)) tr[prefix.encode()] = data.encode() + tr.add(watches.watch_counter_key(StorageNode), watches.ONE_LE64) return True, None diff --git a/simplyblock_core/models/base_model.py b/simplyblock_core/models/base_model.py index 1e2509a4e..4b898d2db 100644 --- a/simplyblock_core/models/base_model.py +++ b/simplyblock_core/models/base_model.py @@ -9,11 +9,18 @@ from pydantic import SecretBytes, SecretStr +from simplyblock_core import watches + class BaseModel(object): _STATUS_CODE_MAP: dict = {} + # When True, write_to_db()/remove() atomically bump watch_seq/ + # in the same FDB transaction so watchers (SSE API) wake up. Plain class + # attribute, not an annotation: must stay out of get_attrs_map()/to_dict(). + _WATCHED = False + id: str = "" uuid: str = "" name: str = "" @@ -197,14 +204,29 @@ def get_last(self, kv_store): return objects[0] return None + @staticmethod + def _write_tx(tr, key, value, counter_key): + tr.set(key, value) + tr.add(counter_key, watches.ONE_LE64) + + @staticmethod + def _remove_tx(tr, key, counter_key): + tr.clear(key) + tr.add(counter_key, watches.ONE_LE64) + def write_to_db(self, kv_store=None): if not kv_store: from simplyblock_core.db_controller import DBController kv_store = DBController().kv_store try: - prefix = self.get_db_id() - st = json.dumps(self.to_dict(unwrap_secrets=True)) - kv_store.set(prefix.encode(), st.encode()) + key = self.get_db_id().encode() + value = json.dumps(self.to_dict(unwrap_secrets=True)).encode() + if self._WATCHED: + import fdb + fdb.transactional(BaseModel._write_tx)( + kv_store, key, value, watches.watch_counter_key(type(self))) + else: + kv_store.set(key, value) return True except Exception: from simplyblock_core import utils @@ -212,8 +234,12 @@ def write_to_db(self, kv_store=None): exit(1) def remove(self, kv_store): - prefix = self.get_db_id() - return kv_store.clear(prefix.encode()) + key = self.get_db_id().encode() + if not self._WATCHED: + return kv_store.clear(key) + import fdb + return fdb.transactional(BaseModel._remove_tx)( + kv_store, key, watches.watch_counter_key(type(self))) def keys(self): return self.get_attrs_map().keys() diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index f06a4997c..e5d2625c1 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -16,6 +16,8 @@ class HashicorpVaultSettings(BaseModel): class Cluster(BaseModel): + _WATCHED = True + STATUS_ACTIVE = "active" STATUS_READONLY = 'read_only' STATUS_INACTIVE = "inactive" diff --git a/simplyblock_core/models/job_schedule.py b/simplyblock_core/models/job_schedule.py index 859f5fcfb..8de7e02a7 100644 --- a/simplyblock_core/models/job_schedule.py +++ b/simplyblock_core/models/job_schedule.py @@ -6,6 +6,8 @@ class JobSchedule(BaseModel): + _WATCHED = True + STATUS_NEW = 'new' STATUS_RUNNING = 'running' STATUS_SUSPENDED = 'suspended' diff --git a/simplyblock_core/models/lvol_model.py b/simplyblock_core/models/lvol_model.py index ea658eb75..65e8880e4 100644 --- a/simplyblock_core/models/lvol_model.py +++ b/simplyblock_core/models/lvol_model.py @@ -7,6 +7,8 @@ class LVol(BaseModel): + _WATCHED = True + STATUS_IN_CREATION = 'in_creation' STATUS_ONLINE = 'online' STATUS_OFFLINE = 'offline' diff --git a/simplyblock_core/models/pool.py b/simplyblock_core/models/pool.py index cf3f19f50..81d77c6f9 100644 --- a/simplyblock_core/models/pool.py +++ b/simplyblock_core/models/pool.py @@ -9,6 +9,8 @@ class Pool(BaseModel): + _WATCHED = True + STATUS_ACTIVE = "active" STATUS_INACTIVE = "inactive" diff --git a/simplyblock_core/models/snapshot.py b/simplyblock_core/models/snapshot.py index 46559e8d0..62002ce1e 100644 --- a/simplyblock_core/models/snapshot.py +++ b/simplyblock_core/models/snapshot.py @@ -6,6 +6,8 @@ class SnapShot(BaseModel): + _WATCHED = True + STATUS_ONLINE = 'online' STATUS_OFFLINE = 'offline' STATUS_IN_DELETION = 'in_deletion' diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 21d22a701..d7c4b76b3 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -21,6 +21,8 @@ class StorageNode(BaseNodeObject): + _WATCHED = True + # Restart phase constants (per-LVS) RESTART_PHASE_PRE_BLOCK = "pre_block" RESTART_PHASE_BLOCKED = "blocked" diff --git a/simplyblock_core/watches.py b/simplyblock_core/watches.py new file mode 100644 index 000000000..4e6f37820 --- /dev/null +++ b/simplyblock_core/watches.py @@ -0,0 +1,33 @@ +# coding=utf-8 +"""Change-counter keys for entity watches. + +Every write/remove of a watched model class atomically bumps a per-class +counter key (``watch_seq/``) in the same FoundationDB transaction +as the entity mutation. Watchers (the web API's SSE layer) set FDB watches on +these counter keys to learn that *something* of that class changed, then +re-read the entity range and diff. + +The ``watch_seq/`` namespace is disjoint from all entity range scans: +``BaseModel.read_from_db`` only scans ``//`` prefixes. + +Stdlib-only leaf module — importable from models without cycles. +""" + +import struct + +WATCH_SEQ_PREFIX = 'watch_seq/' + +# Operand for FDB's atomic ADD mutation: little-endian 64-bit integer 1. +ONE_LE64 = struct.pack(' bytes: + """FDB key of the per-class change counter, e.g. ``b'watch_seq/StorageNode'``.""" + return (WATCH_SEQ_PREFIX + model_cls.__name__).encode() + + +def unpack_counter(raw) -> int: + """Decode a counter value produced by atomic ADD; absent/``None`` -> 0.""" + if not raw: + return 0 + return struct.unpack('`` in the same FDB transaction as the entity mutation +(see ``simplyblock_core.watches``). One shared :class:`EntityWatchHub` per +model class owns a single FDB watch on that counter key; when it fires, the +hub re-reads the class's entity range once and fans the parsed state out to +every subscribed stream. Scoping (e.g. volumes of one pool) happens per +subscription, via the same filter its list endpoint applies — N scoped +subscribers cost one watch and one shared range read per change. + +The hub registers the watch *before* reading the range: any write not visible +to the range read has a commit version at or after the watch's read version +and therefore fires it, so no update is lost. A reconcile timeout re-reads +the range even without a watch fire, bounding staleness for writes from +pre-upgrade components that don't bump counters. +""" + +import asyncio +import json +import logging +import threading +import time +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type + +import fdb +from fastapi import HTTPException, Query +from sse_starlette import EventSourceResponse, ServerSentEvent +from starlette.concurrency import run_in_threadpool +from typing_extensions import Annotated + +from simplyblock_core import watches +from simplyblock_core.db_controller import DBController + +logger = logging.getLogger(__name__) + +# Raw FDB key -> parsed entity dict. Keyed by the FDB key (not the uuid +# field): unique by construction and indifferent to compound hierarchical ids +# (JobSchedule keys are object/JobSchedule///). +RawState = Dict[Any, dict] + +# Filter applied per subscription; must match the corresponding list endpoint. +Projection = Callable[[RawState], RawState] + +# Builds the wire DTO from a parsed entity dict. May do blocking DB reads — +# only ever called via run_in_threadpool. +DTOBuilder = Callable[[dict], Any] + +# Upper bound on staleness for entity writes that don't bump the change +# counter (processes from before the counter was introduced). +WATCH_RECONCILE_SEC = 30.0 + +# Streams are closed after this long; bearer tokens are only validated at +# request start, so this bounds how long a revoked token keeps a live stream +# (clients re-authenticate on reconnect, like Kubernetes watch timeouts). +WATCH_MAX_LIFETIME_SEC = 3600.0 + +_FDB_RETRY_BACKOFF_SEC = (1.0, 2.0, 4.0) +SSE_RETRY_MS = 3000 +PING_SEC = 15 + +_UNSET = object() + +# OpenAPI ``responses`` snippet for routes that also serve ``?watch=true``. +WATCH_RESPONSES: Dict[Any, Any] = { + 200: {'content': {'text/event-stream': {'schema': {'type': 'string'}}}}, +} + +# Query-parameter documentation, shared by all watchable routes. +WATCH_PARAM_DESCRIPTION = ( + 'Stream state changes as Server-Sent Events instead of returning a plain ' + 'response: a `snapshot` event with the current state first, then ' + '`created`/`updated`/`deleted` events carrying the full resource ' + 'representation. Streams do not support resume; reconnecting clients ' + 'receive a fresh snapshot. Changes written by pre-upgrade components may ' + 'take up to 30 seconds to appear.' +) + +# ``watch: WatchParam = False`` on a route adds the documented query flag. +WatchParam = Annotated[bool, Query(description=WATCH_PARAM_DESCRIPTION)] + + +class _Subscription: + """Coalescing mailbox: written by the hub thread, read by one stream.""" + + def __init__(self, loop: asyncio.AbstractEventLoop, notify: asyncio.Event) -> None: + self._loop = loop + self._notify = notify + self._latest: Any = _UNSET + self._dirty = False + self.failed = False + + def _deliver(self, state: RawState) -> None: # event-loop thread + self._latest = state + self._dirty = True + self._notify.set() + + def _fail(self) -> None: # event-loop thread + self.failed = True + self._notify.set() + + def deliver(self, state: RawState) -> None: # hub thread + self._loop.call_soon_threadsafe(self._deliver, state) + + def fail(self) -> None: # hub thread + self._loop.call_soon_threadsafe(self._fail) + + def take(self) -> Any: + """Latest state if there is an unconsumed publication, else ``_UNSET``.""" + if not self._dirty: + return _UNSET + self._dirty = False + return self._latest + + +def _watch_tx(tr: Any, key: bytes) -> Any: + return tr.watch(key) + + +class EntityWatchHub: + """Single FDB watch + shared range re-read for one watched model class.""" + + def __init__(self, model_cls: Type[Any]) -> None: + instance = model_cls() + self._name = model_cls.__name__ + # Built from parts rather than get_db_id(): a fresh JobSchedule's + # compound get_id() is not empty, which would corrupt the prefix. + self._prefix = f'{instance.object_type}/{instance.name}/'.encode() + self._counter_key = watches.watch_counter_key(model_cls) + self._lock = threading.Lock() + self._subs: set = set() + self._cache: Any = _UNSET + self._stop = threading.Event() + self._wake = threading.Event() + self._thread: Optional[threading.Thread] = None + + def subscribe(self, loop: asyncio.AbstractEventLoop, notify: asyncio.Event) -> _Subscription: + sub = _Subscription(loop, notify) + with self._lock: + alive = self._thread is not None and self._thread.is_alive() and not self._stop.is_set() + if not alive: + self._cache = _UNSET + self._stop = threading.Event() + self._wake = threading.Event() + self._thread = threading.Thread( + target=self._run, args=(self._stop, self._wake), + daemon=True, name=f'watch-{self._name}', + ) + self._thread.start() + self._subs.add(sub) + if self._cache is not _UNSET: + # subscribe() runs on the event loop thread, so deliver directly. + sub._deliver(self._cache) + return sub + + def unsubscribe(self, sub: _Subscription) -> None: + with self._lock: + self._subs.discard(sub) + if not self._subs: + self._stop.set() + self._wake.set() + + def subscriber_count(self) -> int: + with self._lock: + return len(self._subs) + + def _publish(self, state: RawState) -> None: + with self._lock: + self._cache = state + subs = list(self._subs) + for sub in subs: + sub.deliver(state) + + def _fail_all(self) -> None: + with self._lock: + subs = list(self._subs) + for sub in subs: + sub.fail() + + def _read_range(self, kv_store: Any) -> RawState: + state: RawState = {} + for k, v in kv_store.get_range_startswith(self._prefix): + try: + state[bytes(k)] = json.loads(v) + except ValueError: + logger.warning('Skipping unparsable record %s', bytes(k)) + return state + + def _run(self, stop: threading.Event, wake: threading.Event) -> None: + kv_store = DBController().kv_store + set_watch = fdb.transactional(_watch_tx) + failures = 0 + while not stop.is_set(): + watch = None + try: + # Watch first, then read: see module docstring. + watch = set_watch(kv_store, self._counter_key) + state = self._read_range(kv_store) + failures = 0 + with self._lock: + changed = self._cache is _UNSET or state != self._cache + if changed: + self._publish(state) + wake.clear() + # Runs on the FDB network thread: must only set the event, + # never block or call back into fdb. + watch.on_ready(lambda _f: wake.set()) + wake.wait(timeout=WATCH_RECONCILE_SEC) + except Exception: + logger.exception('Watch hub for %s failed', self._name) + failures += 1 + if failures > len(_FDB_RETRY_BACKOFF_SEC): + self._fail_all() + return + stop.wait(timeout=_FDB_RETRY_BACKOFF_SEC[failures - 1]) + finally: + if watch is not None: + try: + watch.cancel() + except Exception: + pass + + +_hubs: Dict[type, EntityWatchHub] = {} +_hubs_lock = threading.Lock() + + +def get_hub(model_cls: Type[Any]) -> EntityWatchHub: + with _hubs_lock: + hub = _hubs.get(model_cls) + if hub is None: + hub = _hubs[model_cls] = EntityWatchHub(model_cls) + return hub + + +def _build_snapshot( + filtered: RawState, build_dto: DTOBuilder, single: bool, +) -> Tuple[List[ServerSentEvent], Dict[Any, str]]: + cache = {key: build_dto(entity).model_dump_json() for key, entity in filtered.items()} + if single: + if not cache: + return [], cache + data = next(iter(cache.values())) + else: + data = '[' + ','.join(cache[key] for key in sorted(cache)) + ']' + return [ServerSentEvent(event='snapshot', data=data, retry=SSE_RETRY_MS)], cache + + +def _build_diff( + prev: RawState, new: RawState, cache: Dict[Any, str], + build_dto: DTOBuilder, single: bool, +) -> Tuple[List[ServerSentEvent], Dict[Any, str]]: + events: List[ServerSentEvent] = [] + new_cache: Dict[Any, str] = {} + for key, entity in new.items(): + if key not in prev: + new_cache[key] = build_dto(entity).model_dump_json() + events.append(ServerSentEvent(event='updated' if single else 'created', data=new_cache[key])) + elif entity != prev[key]: + new_cache[key] = build_dto(entity).model_dump_json() + # Suppress changes invisible in the DTO (e.g. JobSchedule's + # updated_at lease heartbeat rewrites). + if new_cache[key] != cache.get(key): + events.append(ServerSentEvent(event='updated', data=new_cache[key])) + else: + new_cache[key] = cache[key] + for key in prev: + if key not in new: + events.append(ServerSentEvent(event='deleted', data=cache.get(key, '{}'))) + return events, new_cache + + +def watch_response( + model_cls: Type[Any], + project: Projection, + build_dto: DTOBuilder, + single_id: Optional[str] = None, + ancestors: Sequence[Tuple[Type[Any], str]] = (), +) -> EventSourceResponse: + """SSE stream of state changes for one watched entity class. + + ``project`` must apply the same filter as the corresponding list endpoint; + the diff runs on its output, so entities entering/leaving the filtered set + emit ``created``/``deleted``. ``single_id`` switches to detail semantics + (single-DTO snapshot, close after ``deleted``). ``ancestors`` are + ``(model_cls, id)`` pairs from the route's dependency chain: the stream + closes when any of them disappears, so a reconnect gets the endpoint's + regular 404. + """ + if DBController().kv_store is None: + raise HTTPException(503, 'Database unavailable') + + single = single_id is not None + ancestor_specs = [(cls, str(ancestor_id)) for cls, ancestor_id in ancestors] + + async def event_stream() -> Any: + loop = asyncio.get_running_loop() + notify = asyncio.Event() + main_sub = get_hub(model_cls).subscribe(loop, notify) + ancestor_subs = [ + (get_hub(cls).subscribe(loop, notify), cls, ancestor_id) + for cls, ancestor_id in ancestor_specs + ] + try: + prev: Optional[RawState] = None + dto_cache: Dict[Any, str] = {} + deadline = time.monotonic() + WATCH_MAX_LIFETIME_SEC + while True: + timeout = deadline - time.monotonic() + if timeout <= 0: + return + try: + await asyncio.wait_for(notify.wait(), timeout) + except asyncio.TimeoutError: + return + notify.clear() + + if main_sub.failed or any(sub.failed for sub, _, _ in ancestor_subs): + yield ServerSentEvent(event='error', data='{"detail": "backend unavailable"}') + return + + for ancestor_sub, _, ancestor_id in ancestor_subs: + ancestor_state = ancestor_sub.take() + if ancestor_state is _UNSET: + continue + if not any(entity.get('uuid') == ancestor_id for entity in ancestor_state.values()): + return + + state = main_sub.take() + if state is _UNSET: + continue + filtered = await run_in_threadpool(project, state) + if prev is None: + if single and not filtered: + # Deleted between dependency resolution and first + # publication; close so the reconnect 404s. + return + events, dto_cache = await run_in_threadpool( + _build_snapshot, filtered, build_dto, single) + else: + events, dto_cache = await run_in_threadpool( + _build_diff, prev, filtered, dto_cache, build_dto, single) + prev = filtered + for event in events: + yield event + if single and not filtered: + return + finally: + get_hub(model_cls).unsubscribe(main_sub) + for ancestor_sub, cls, _ in ancestor_subs: + get_hub(cls).unsubscribe(ancestor_sub) + + return EventSourceResponse( + event_stream(), + ping=PING_SEC, + headers={'Cache-Control': 'no-store', 'X-Accel-Buffering': 'no'}, + ) diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index 7cc78f0f7..aa14e2d68 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -1,12 +1,14 @@ from threading import Thread -from typing import Annotated, List, Literal, Optional +from typing import Annotated, List, Literal, Optional, Union from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response from pydantic import BaseModel, Field, SecretStr from pydantic.networks import AnyUrl, UrlConstraints +from sse_starlette import EventSourceResponse from simplyblock_core.db_controller import DBController +from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.cluster import HashicorpVaultSettings as ModelVaultSettings from simplyblock_core import cluster_ops @@ -16,6 +18,7 @@ from .storage_node import api as storage_node_api from .task import api as task_api from .._dtos import ClusterDTO +from .._watch import WATCH_RESPONSES, WatchParam, watch_response from .. import util as util @@ -87,8 +90,16 @@ class ClusterParams(BaseModel): hashicorp_vault_settings: Optional[HashicorpVaultSettings] = None -@api.get('/', name='clusters:list') -def list() -> List[ClusterDTO]: +def _cluster_dto(data: dict) -> ClusterDTO: + cluster = ClusterModel(data) + ret = db.get_cluster_capacity(cluster, 1) + return ClusterDTO.from_model(cluster, ret[0] if ret else None) + + +@api.get('/', name='clusters:list', response_model=List[ClusterDTO], responses=WATCH_RESPONSES) +def list(watch: WatchParam = False) -> Union[List[ClusterDTO], EventSourceResponse]: + if watch: + return watch_response(ClusterModel, lambda state: state, _cluster_dto) data = [] for cluster in db.get_clusters(): stat_obj = None @@ -127,8 +138,16 @@ def add(request: Request, parameters: ClusterParams, response_format: util.Creat instance_api = APIRouter(prefix='/{cluster_id}') -@instance_api.get('/', name='clusters:detail') -def get(cluster: Cluster) -> ClusterDTO: +@instance_api.get('/', name='clusters:detail', response_model=ClusterDTO, responses=WATCH_RESPONSES) +def get(cluster: Cluster, watch: WatchParam = False) -> Union[ClusterDTO, EventSourceResponse]: + if watch: + cluster_id = cluster.get_id() + return watch_response( + ClusterModel, + lambda state: {k: d for k, d in state.items() if d.get('uuid') == cluster_id}, + _cluster_dto, + single_id=cluster_id, + ) stat_obj = None ret = db.get_cluster_capacity(cluster, 1) if ret: diff --git a/simplyblock_web/api/v2/cluster/storage_node/__init__.py b/simplyblock_web/api/v2/cluster/storage_node/__init__.py index 942242c99..278095ef1 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_node/__init__.py @@ -1,16 +1,20 @@ from threading import Thread -from typing import List, Optional +from typing import List, Optional, Union from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response from pydantic import BaseModel, Field +from sse_starlette import EventSourceResponse from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import tasks_controller +from simplyblock_core.models.cluster import Cluster as ClusterModel +from simplyblock_core.models.storage_node import StorageNode as StorageNodeModel from simplyblock_core import storage_node_ops from ... import util as util from ..._dependencies import Cluster, StorageNode +from ..._watch import WATCH_RESPONSES, WatchParam, watch_response from .device import api as device_api from ..._dtos import StorageNodeDTO, TaskDTO @@ -19,8 +23,22 @@ db = DBController() -@api.get('/', name='clusters:storage-nodes:list') -def list(cluster: Cluster) -> List[StorageNodeDTO]: +def _storage_node_dto(data: dict) -> StorageNodeDTO: + storage_node = StorageNodeModel(data) + ret = db.get_node_capacity(storage_node, 1) + return StorageNodeDTO.from_model(storage_node, ret[0] if ret else None) + + +@api.get('/', name='clusters:storage-nodes:list', response_model=List[StorageNodeDTO], responses=WATCH_RESPONSES) +def list(cluster: Cluster, watch: WatchParam = False) -> Union[List[StorageNodeDTO], EventSourceResponse]: + if watch: + cluster_id = cluster.get_id() + return watch_response( + StorageNodeModel, + lambda state: {k: d for k, d in state.items() if d.get('cluster_id') == cluster_id}, + _storage_node_dto, + ancestors=[(ClusterModel, cluster_id)], + ) data = [] for storage_node in db.get_storage_nodes_by_cluster_id(cluster.get_id()): node_stat_obj = None @@ -99,8 +117,17 @@ def add(request: Request, cluster: Cluster, parameters: StorageNodeParams, respo instance_api = APIRouter(prefix='/{storage_node_id}') -@instance_api.get('/', name='clusters:storage-nodes:detail') -def get(cluster: Cluster, storage_node: StorageNode): +@instance_api.get('/', name='clusters:storage-nodes:detail', response_model=StorageNodeDTO, responses=WATCH_RESPONSES) +def get(cluster: Cluster, storage_node: StorageNode, watch: WatchParam = False) -> Union[StorageNodeDTO, EventSourceResponse]: + if watch: + node_id = storage_node.get_id() + return watch_response( + StorageNodeModel, + lambda state: {k: d for k, d in state.items() if d.get('uuid') == node_id}, + _storage_node_dto, + single_id=node_id, + ancestors=[(ClusterModel, cluster.get_id())], + ) node_stat_obj = None ret = db.get_node_capacity(storage_node, 1) if ret: diff --git a/simplyblock_web/api/v2/cluster/storage_node/device.py b/simplyblock_web/api/v2/cluster/storage_node/device.py index 6fe7a60ae..6bb51d4b0 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/device.py +++ b/simplyblock_web/api/v2/cluster/storage_node/device.py @@ -1,20 +1,56 @@ -from typing import List, Optional +from typing import Callable, List, Optional, Union from fastapi import APIRouter, Response +from sse_starlette import EventSourceResponse from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import device_controller +from simplyblock_core.models.cluster import Cluster as ClusterModel +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode as StorageNodeModel from ..._dependencies import Cluster, StorageNode, Device from ..._dtos import DeviceDTO +from ..._watch import RawState, WATCH_RESPONSES, WatchParam, watch_response api = APIRouter() db = DBController() -@api.get('/', name='clusters:storage_nodes:devices:list') -def list(cluster: Cluster, storage_node: StorageNode) -> List[DeviceDTO]: +def _make_device_dto(storage_node_id: str) -> Callable[[dict], DeviceDTO]: + def build(data: dict) -> DeviceDTO: + device = NVMeDevice(data) + ret = db.get_device_stats(device, 1) + return DeviceDTO.from_model(device, storage_node_id, ret[0] if ret else None) + return build + + +def _make_device_projection(node_id: str, device_id: Optional[str] = None) -> Callable[[RawState], RawState]: + # Devices are embedded in their StorageNode record; explode them into + # per-device entries so the diff yields device-level events. + def project(state: RawState) -> RawState: + for node in state.values(): + if node.get('uuid') == node_id: + return { + device['uuid']: device + for device in node.get('nvme_devices', []) + if device_id is None or device.get('uuid') == device_id + } + return {} + return project + + +@api.get('/', name='clusters:storage_nodes:devices:list', response_model=List[DeviceDTO], responses=WATCH_RESPONSES) +def list(cluster: Cluster, storage_node: StorageNode, watch: WatchParam = False) -> Union[List[DeviceDTO], EventSourceResponse]: + if watch: + node_id = storage_node.get_id() + return watch_response( + StorageNodeModel, + _make_device_projection(node_id), + _make_device_dto(node_id), + ancestors=[(ClusterModel, cluster.get_id()), (StorageNodeModel, node_id)], + ) data = [] for device in storage_node.nvme_devices: stat_obj = None @@ -28,8 +64,17 @@ def list(cluster: Cluster, storage_node: StorageNode) -> List[DeviceDTO]: instance_api = APIRouter(prefix='/{device_id}') -@instance_api.get('/', name='clusters:storage_nodes:devices:detail') -def get(cluster: Cluster, storage_node: StorageNode, device: Device) -> DeviceDTO: +@instance_api.get('/', name='clusters:storage_nodes:devices:detail', response_model=DeviceDTO, responses=WATCH_RESPONSES) +def get(cluster: Cluster, storage_node: StorageNode, device: Device, watch: WatchParam = False) -> Union[DeviceDTO, EventSourceResponse]: + if watch: + node_id = storage_node.get_id() + return watch_response( + StorageNodeModel, + _make_device_projection(node_id, device.get_id()), + _make_device_dto(node_id), + single_id=device.get_id(), + ancestors=[(ClusterModel, cluster.get_id()), (StorageNodeModel, node_id)], + ) stat_obj = None ret = db.get_device_stats(device, 1) if ret: diff --git a/simplyblock_web/api/v2/cluster/storage_pool/__init__.py b/simplyblock_web/api/v2/cluster/storage_pool/__init__.py index f9627d420..afbe47435 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/__init__.py @@ -1,16 +1,19 @@ -from typing import Annotated, List, Optional +from typing import Annotated, List, Optional, Union from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response from pydantic import BaseModel, Field +from sse_starlette import EventSourceResponse from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import pool_controller from simplyblock_core import utils as core_utils +from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.pool import Pool as PoolModel from ... import util as util from ..._dependencies import Cluster, StoragePool +from ..._watch import WATCH_RESPONSES, WatchParam, watch_response from .volume import api as volume_api from .snapshot import api as snapshot_api from ..._dtos import StoragePoolDTO @@ -20,8 +23,20 @@ db = DBController() -@api.get('/', name='clusters:storage-pools:list') -def list(cluster: Cluster) -> List[StoragePoolDTO]: +def _pool_dto(data: dict) -> StoragePoolDTO: + return StoragePoolDTO.from_model(PoolModel(data), None) + + +@api.get('/', name='clusters:storage-pools:list', response_model=List[StoragePoolDTO], responses=WATCH_RESPONSES) +def list(cluster: Cluster, watch: WatchParam = False) -> Union[List[StoragePoolDTO], EventSourceResponse]: + if watch: + cluster_id = cluster.get_id() + return watch_response( + PoolModel, + lambda state: {k: d for k, d in state.items() if d.get('cluster_id') == cluster_id}, + _pool_dto, + ancestors=[(ClusterModel, cluster_id)], + ) return [StoragePoolDTO.from_model(pool, None) for pool in db.get_pools(cluster.get_id())] @@ -68,8 +83,17 @@ def add(request: Request, cluster: Cluster, parameters: StoragePoolParams, respo instance_api = APIRouter(prefix='/{pool_id}') -@instance_api.get('/', name='clusters:storage-pools:detail') -def get(cluster: Cluster, pool: StoragePool) -> StoragePoolDTO: +@instance_api.get('/', name='clusters:storage-pools:detail', response_model=StoragePoolDTO, responses=WATCH_RESPONSES) +def get(cluster: Cluster, pool: StoragePool, watch: WatchParam = False) -> Union[StoragePoolDTO, EventSourceResponse]: + if watch: + pool_id = pool.get_id() + return watch_response( + PoolModel, + lambda state: {k: d for k, d in state.items() if d.get('uuid') == pool_id}, + _pool_dto, + single_id=pool_id, + ancestors=[(ClusterModel, cluster.get_id())], + ) stat_obj = None return StoragePoolDTO.from_model(pool, stat_obj) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/snapshot.py b/simplyblock_web/api/v2/cluster/storage_pool/snapshot.py index 96d091687..be7372976 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/snapshot.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/snapshot.py @@ -1,20 +1,39 @@ -from typing import List +from typing import Callable, List, Union from fastapi import APIRouter, Response, Request +from sse_starlette import EventSourceResponse from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import snapshot_controller +from simplyblock_core.models.cluster import Cluster as ClusterModel +from simplyblock_core.models.pool import Pool as PoolModel +from simplyblock_core.models.snapshot import SnapShot as SnapshotModel from ..._dependencies import Cluster, StoragePool, Snapshot from ..._dtos import SnapshotDTO +from ..._watch import WATCH_RESPONSES, WatchParam, watch_response api = APIRouter() db = DBController() -@api.get('/', name='clusters:storage-pools:snapshots:list') -def list(request: Request, cluster: Cluster, pool: StoragePool) -> List[SnapshotDTO]: +def _make_snapshot_dto(request: Request, cluster_id: str, pool_id: str) -> Callable[[dict], SnapshotDTO]: + def build(data: dict) -> SnapshotDTO: + return SnapshotDTO.from_model(SnapshotModel(data), request, cluster_id=cluster_id, pool_id=pool_id) + return build + + +@api.get('/', name='clusters:storage-pools:snapshots:list', response_model=List[SnapshotDTO], responses=WATCH_RESPONSES) +def list(request: Request, cluster: Cluster, pool: StoragePool, watch: WatchParam = False) -> Union[List[SnapshotDTO], EventSourceResponse]: + if watch: + pool_id = pool.get_id() + return watch_response( + SnapshotModel, + lambda state: {k: d for k, d in state.items() if d.get('pool_uuid') == pool_id}, + _make_snapshot_dto(request, cluster.get_id(), pool_id), + ancestors=[(ClusterModel, cluster.get_id()), (PoolModel, pool_id)], + ) return [ SnapshotDTO.from_model(snapshot, request, cluster_id=cluster.get_id(), pool_id=pool.get_id()) for snapshot in db.get_snapshots_by_pool_id(pool.get_id()) @@ -24,8 +43,17 @@ def list(request: Request, cluster: Cluster, pool: StoragePool) -> List[Snapshot instance_api = APIRouter(prefix='/{snapshot_id}') -@instance_api.get('/', name='clusters:storage-pools:snapshots:detail') -def get(request: Request, cluster: Cluster, pool: StoragePool, snapshot: Snapshot) -> SnapshotDTO: +@instance_api.get('/', name='clusters:storage-pools:snapshots:detail', response_model=SnapshotDTO, responses=WATCH_RESPONSES) +def get(request: Request, cluster: Cluster, pool: StoragePool, snapshot: Snapshot, watch: WatchParam = False) -> Union[SnapshotDTO, EventSourceResponse]: + if watch: + snapshot_id = snapshot.get_id() + return watch_response( + SnapshotModel, + lambda state: {k: d for k, d in state.items() if d.get('uuid') == snapshot_id}, + _make_snapshot_dto(request, cluster.get_id(), pool.get_id()), + single_id=snapshot_id, + ancestors=[(ClusterModel, cluster.get_id()), (PoolModel, pool.get_id())], + ) return SnapshotDTO.from_model(snapshot, request, cluster_id=cluster.get_id(), pool_id=pool.get_id()) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py index a2dbdedc0..ca899c0d4 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py @@ -3,14 +3,18 @@ from fastapi import APIRouter, HTTPException, Request, Response from pydantic import BaseModel, Field, RootModel +from sse_starlette import EventSourceResponse from simplyblock_core.db_controller import DBController from simplyblock_core import utils as core_utils from simplyblock_core.controllers import backup_controller, lvol_controller, snapshot_controller +from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.pool import Pool as PoolModel from ...._dependencies import Cluster, StoragePool, Volume from ...._dtos import BackupDTO, VolumeDTO, SnapshotDTO, TaskDTO +from ...._watch import WATCH_RESPONSES, WatchParam, watch_response from .... import util from .migration import api as migration_api @@ -19,8 +23,21 @@ db = DBController() -@api.get('/', name='clusters:storage-pools:volumes:list') -def list(request: Request, cluster: Cluster, pool: StoragePool) -> List[VolumeDTO]: +@api.get('/', name='clusters:storage-pools:volumes:list', response_model=List[VolumeDTO], responses=WATCH_RESPONSES) +def list(request: Request, cluster: Cluster, pool: StoragePool, watch: WatchParam = False) -> Union[List[VolumeDTO], EventSourceResponse]: + if watch: + pool_id = pool.get_id() + cluster_id = cluster.get_id() + return watch_response( + LVol, + # Same filter as get_lvols_by_pool_id: pool membership, deleted excluded. + lambda state: { + k: d for k, d in state.items() + if d.get('pool_uuid') == pool_id and d.get('status') != LVol.STATUS_DELETED + }, + lambda d: VolumeDTO.from_model(LVol(d), request, cluster_id, None), + ancestors=[(ClusterModel, cluster_id), (PoolModel, pool_id)], + ) data = [] for lvol in db.get_lvols_by_pool_id(pool.get_id()): stat_obj = None @@ -140,8 +157,27 @@ def replicate_lvol_on_source_cluster(cluster: Cluster, pool: StoragePool, body: instance_api = APIRouter(prefix='/{volume_id}') -@instance_api.get('/', name='clusters:storage-pools:volumes:detail') -def get(request: Request, cluster: Cluster, pool: StoragePool, volume: Volume) -> VolumeDTO: +@instance_api.get('/', name='clusters:storage-pools:volumes:detail', response_model=VolumeDTO, responses=WATCH_RESPONSES) +def get(request: Request, cluster: Cluster, pool: StoragePool, volume: Volume, watch: WatchParam = False) -> Union[VolumeDTO, EventSourceResponse]: + if watch: + volume_id = volume.get_id() + cluster_id = cluster.get_id() + + def _volume_dto(d: dict) -> VolumeDTO: + lvol = LVol(d) + rep_info = lvol_controller.get_replication_info(lvol.get_id()) + return VolumeDTO.from_model(lvol, request, cluster_id, None, rep_info) + + return watch_response( + LVol, + lambda state: { + k: d for k, d in state.items() + if d.get('uuid') == volume_id and d.get('status') != LVol.STATUS_DELETED + }, + _volume_dto, + single_id=volume_id, + ancestors=[(ClusterModel, cluster_id), (PoolModel, pool.get_id())], + ) stat_obj = None rep_info = lvol_controller.get_replication_info(volume.get_id()) return VolumeDTO.from_model(volume, request, cluster.get_id(), stat_obj, rep_info) diff --git a/simplyblock_web/api/v2/cluster/task.py b/simplyblock_web/api/v2/cluster/task.py index 42ac831be..d72d70f4c 100644 --- a/simplyblock_web/api/v2/cluster/task.py +++ b/simplyblock_web/api/v2/cluster/task.py @@ -1,20 +1,38 @@ -from typing import List +from typing import List, Union from fastapi import APIRouter +from sse_starlette import EventSourceResponse from simplyblock_core.db_controller import DBController +from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.job_schedule import JobSchedule from .._dependencies import Cluster, Task from .._dtos import TaskDTO +from .._watch import WATCH_RESPONSES, WatchParam, watch_response api = APIRouter() db = DBController() -@api.get('/', name='clusters:tasks:list') -def list(cluster: Cluster) -> List[TaskDTO]: +def _task_dto(data: dict) -> TaskDTO: + return TaskDTO.from_model(JobSchedule(data)) + + +@api.get('/', name='clusters:tasks:list', response_model=List[TaskDTO], responses=WATCH_RESPONSES) +def list(cluster: Cluster, watch: WatchParam = False) -> Union[List[TaskDTO], EventSourceResponse]: + if watch: + cluster_id = cluster.get_id() + return watch_response( + JobSchedule, + lambda state: { + k: d for k, d in state.items() + if d.get('cluster_id') == cluster_id and d.get('function_name') != JobSchedule.FN_DEV_MIG + }, + _task_dto, + ancestors=[(ClusterModel, cluster_id)], + ) cluster_tasks = db.get_job_tasks(cluster.get_id(), limit=0) data = [] for t in cluster_tasks: @@ -27,8 +45,17 @@ def list(cluster: Cluster) -> List[TaskDTO]: instance_api = APIRouter(prefix='/{task_id}') -@instance_api.get('/', name='clusters:tasks:detail') -def get(cluster: Cluster, task: Task) -> TaskDTO: +@instance_api.get('/', name='clusters:tasks:detail', response_model=TaskDTO, responses=WATCH_RESPONSES) +def get(cluster: Cluster, task: Task, watch: WatchParam = False) -> Union[TaskDTO, EventSourceResponse]: + if watch: + task_uuid = task.uuid + return watch_response( + JobSchedule, + lambda state: {k: d for k, d in state.items() if d.get('uuid') == task_uuid}, + _task_dto, + single_id=task_uuid, + ancestors=[(ClusterModel, cluster.get_id())], + ) return TaskDTO.from_model(task) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5d0f08aa1..5ddd4b962 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -22,7 +22,13 @@ def _stub(name, **attrs): if 'fdb' not in sys.modules: class _FDBError(Exception): pass - _stub('fdb', open=lambda *a, **kw: None, FDBError=_FDBError) + # ``transactional`` mirrors the real decorator's calling convention: the + # wrapped function's first argument is the database/transaction. Unit + # tests pass a MagicMock kv_store, so running the body directly against + # it keeps existing ``kv_store.set(...)`` assertions working while + # watched-model writes land their counter ``add(...)`` on the same mock. + _stub('fdb', open=lambda *a, **kw: None, FDBError=_FDBError, + transactional=lambda f: f) _stub('fdb.tuple') diff --git a/tests/unit/test_watch_counters.py b/tests/unit/test_watch_counters.py new file mode 100644 index 000000000..0e13579e3 --- /dev/null +++ b/tests/unit/test_watch_counters.py @@ -0,0 +1,106 @@ +# coding=utf-8 +"""Change-counter keys and the transactional write+bump in BaseModel.""" + +import struct +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core import watches +from simplyblock_core.models.base_model import BaseModel +from simplyblock_core.models.cluster import Cluster, ClusterAddNodeLock, PortReservation +from simplyblock_core.models.events import EventObj +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.lvol_model import LVol, LVolMini +from simplyblock_core.models.pool import Pool +from simplyblock_core.models.snapshot import SnapShot, SnapShotMini +from simplyblock_core.models.stats import LVolStatObject, StatsObject +from simplyblock_core.models.storage_node import StorageNode + + +WATCHED_CLASSES = [Cluster, StorageNode, Pool, LVol, SnapShot, JobSchedule] +UNWATCHED_CLASSES = [ + EventObj, LVolMini, SnapShotMini, StatsObject, LVolStatObject, + ClusterAddNodeLock, PortReservation, +] + + +def test_counter_key_format(): + assert watches.watch_counter_key(StorageNode) == b'watch_seq/StorageNode' + assert watches.watch_counter_key(LVol) == b'watch_seq/LVol' + + +@pytest.mark.parametrize('model_cls', WATCHED_CLASSES) +def test_counter_keys_disjoint_from_entity_scans(model_cls): + """read_from_db scans '//' prefixes; counter keys + must never be picked up by any such scan.""" + counter_key = watches.watch_counter_key(model_cls) + for other_cls in WATCHED_CLASSES + UNWATCHED_CLASSES: + instance = other_cls() + prefix = f'{instance.object_type}/{instance.name}/'.encode() + assert not counter_key.startswith(prefix) + + +@pytest.mark.parametrize('model_cls', WATCHED_CLASSES) +def test_watched_flag_set(model_cls): + assert model_cls._WATCHED is True + + +@pytest.mark.parametrize('model_cls', UNWATCHED_CLASSES) +def test_unwatched_flag_not_set(model_cls): + assert model_cls._WATCHED is False + + +def test_watched_write_bumps_counter(): + kv = MagicMock() + pool = Pool({'uuid': 'pool-1'}) + pool.write_to_db(kv) + kv.set.assert_called_once() + key, _value = kv.set.call_args[0] + assert key == b'object/Pool/pool-1' + kv.add.assert_called_once_with(b'watch_seq/Pool', watches.ONE_LE64) + + +def test_unwatched_write_does_not_bump(): + kv = MagicMock() + event = EventObj({'uuid': 'ev-1'}) + event.write_to_db(kv) + kv.set.assert_called_once() + kv.add.assert_not_called() + + +def test_watched_remove_bumps_counter(): + kv = MagicMock() + pool = Pool({'uuid': 'pool-1'}) + pool.remove(kv) + kv.clear.assert_called_once_with(b'object/Pool/pool-1') + kv.add.assert_called_once_with(b'watch_seq/Pool', watches.ONE_LE64) + + +def test_unwatched_remove_does_not_bump(): + kv = MagicMock() + event = EventObj({'uuid': 'ev-1', 'cluster_uuid': 'c', 'date': 5}) + event.remove(kv) + kv.clear.assert_called_once() + kv.add.assert_not_called() + + +def test_compound_key_write_bumps_class_counter(): + """JobSchedule's compound get_id() must not leak into the counter key.""" + kv = MagicMock() + task = JobSchedule({'uuid': 'task-1', 'cluster_id': 'cl-1', 'date': 42}) + task.write_to_db(kv) + key, _value = kv.set.call_args[0] + assert key == b'object/JobSchedule/cl-1/42/task-1' + kv.add.assert_called_once_with(b'watch_seq/JobSchedule', watches.ONE_LE64) + + +def test_base_model_default_unwatched(): + assert BaseModel._WATCHED is False + + +def test_unpack_counter(): + assert watches.unpack_counter(None) == 0 + assert watches.unpack_counter(b'') == 0 + assert watches.unpack_counter(struct.pack(' _DTO: + return _DTO(uuid=entity['uuid'], status=entity.get('status', '')) + + +# ---- _build_snapshot / _build_diff (pure functions) ---- + +def test_snapshot_collection_is_json_array(): + events, cache = _build_snapshot( + {b'k1': {'uuid': 'a'}, b'k2': {'uuid': 'b'}}, _dto, single=False) + [event] = events + assert event.event == 'snapshot' + assert event.data.startswith('[') and event.data.endswith(']') + assert '"a"' in event.data and '"b"' in event.data + assert set(cache) == {b'k1', b'k2'} + + +def test_snapshot_single_is_bare_object(): + events, _ = _build_snapshot({b'k1': {'uuid': 'a'}}, _dto, single=True) + [event] = events + assert event.event == 'snapshot' + assert event.data.startswith('{') + + +def test_diff_created_updated_deleted(): + prev = {b'k1': {'uuid': 'a', 'status': 'x'}, b'k2': {'uuid': 'b', 'status': 'x'}} + _, cache = _build_snapshot(prev, _dto, single=False) + new = {b'k2': {'uuid': 'b', 'status': 'y'}, b'k3': {'uuid': 'c', 'status': 'x'}} + events, new_cache = _build_diff(prev, new, cache, _dto, single=False) + by_name = {e.event: e.data for e in events} + assert set(by_name) == {'created', 'updated', 'deleted'} + assert '"c"' in by_name['created'] + assert '"b"' in by_name['updated'] and '"y"' in by_name['updated'] + assert '"a"' in by_name['deleted'] # last-known DTO + assert set(new_cache) == {b'k2', b'k3'} + + +def test_diff_suppresses_changes_invisible_in_dto(): + """Raw dict changed (lease heartbeat) but the DTO is identical -> no event.""" + task = { + 'uuid': str(uuid4()), 'cluster_id': str(uuid4()), 'date': 1, + 'function_name': 'node_restart', 'status': 'running', + 'updated_at': '2026-01-01T00:00:00', + } + build = lambda d: TaskDTO.from_model(JobSchedule(d)) # noqa: E731 + prev = {b'k1': task} + _, cache = _build_snapshot(prev, build, single=False) + new = {b'k1': {**task, 'updated_at': '2026-01-01T00:00:05'}} + events, _ = _build_diff(prev, new, cache, build, single=False) + assert events == [] + + +def test_diff_compound_keys(): + """JobSchedule FDB keys share a cluster prefix; diff must key exactly.""" + cluster_id = str(uuid4()) + t1 = {'uuid': str(uuid4()), 'cluster_id': cluster_id, 'date': 1, + 'function_name': 'node_restart', 'status': 'running'} + t2 = {'uuid': str(uuid4()), 'cluster_id': cluster_id, 'date': 2, + 'function_name': 'node_restart', 'status': 'running'} + build = lambda d: TaskDTO.from_model(JobSchedule(d)) # noqa: E731 + k1 = f'object/JobSchedule/{cluster_id}/1/{t1["uuid"]}'.encode() + k2 = f'object/JobSchedule/{cluster_id}/2/{t2["uuid"]}'.encode() + prev = {k1: t1, k2: t2} + _, cache = _build_snapshot(prev, build, single=False) + new = {k1: t1, k2: {**t2, 'status': 'done'}} + events, _ = _build_diff(prev, new, cache, build, single=False) + [event] = events + assert event.event == 'updated' + assert t2['uuid'] in event.data + + +def test_diff_single_mode_maps_created_to_updated(): + events, _ = _build_diff({}, {b'k1': {'uuid': 'a'}}, {}, _dto, single=True) + [event] = events + assert event.event == 'updated' + + +def test_diff_scope_leave_emits_deleted(): + """An entity leaving the projected set (filtered out) emits deleted.""" + project = lambda state: { # noqa: E731 + k: d for k, d in state.items() if d.get('status') != 'deleted'} + s1 = project({b'k1': {'uuid': 'a', 'status': 'online'}}) + _, cache = _build_snapshot(s1, _dto, single=False) + s2 = project({b'k1': {'uuid': 'a', 'status': 'deleted'}}) + events, _ = _build_diff(s1, s2, cache, _dto, single=False) + [event] = events + assert event.event == 'deleted' + + +# ---- generator behavior with a fake hub ---- + +class FakeHub: + def __init__(self): + self.subs = [] + self.cache = None + + def subscribe(self, loop, notify): + sub = _Subscription(loop, notify) + self.subs.append(sub) + if self.cache is not None: + sub._deliver(self.cache) + return sub + + def unsubscribe(self, sub): + self.subs.remove(sub) + + def publish(self, state): + self.cache = state + for sub in list(self.subs): + sub._deliver(state) + + def fail(self): + for sub in list(self.subs): + sub._fail() + + +class Thing: + pass + + +class Parent: + pass + + +def _patch(monkeypatch, hubs): + monkeypatch.setattr(_watch, 'DBController', lambda: SimpleNamespace(kv_store=object())) + monkeypatch.setattr(_watch, 'get_hub', lambda cls: hubs[cls]) + + +async def _next_event(gen, timeout=2.0): + return await asyncio.wait_for(gen.__anext__(), timeout) + + +def test_stream_snapshot_update_delete(monkeypatch): + hub = FakeHub() + _patch(monkeypatch, {Thing: hub}) + hub.publish({b'k1': {'uuid': 'a', 'status': 'online'}}) + + async def scenario(): + response = watch_response(Thing, lambda s: s, _dto) + gen = response.body_iterator + event = await _next_event(gen) + assert event.event == 'snapshot' + + hub.publish({b'k1': {'uuid': 'a', 'status': 'offline'}}) + event = await _next_event(gen) + assert event.event == 'updated' and 'offline' in event.data + + hub.publish({}) + event = await _next_event(gen) + assert event.event == 'deleted' + + await gen.aclose() + assert hub.subs == [] + + asyncio.run(scenario()) + + +def test_stream_coalesces_bursts(monkeypatch): + hub = FakeHub() + _patch(monkeypatch, {Thing: hub}) + hub.publish({b'k1': {'uuid': 'a', 'status': 's0'}}) + + async def scenario(): + gen = watch_response(Thing, lambda s: s, _dto).body_iterator + assert (await _next_event(gen)).event == 'snapshot' + + # Two publications without yielding to the generator: only the + # latest state is diffed. + hub.publish({b'k1': {'uuid': 'a', 'status': 's1'}}) + hub.publish({b'k1': {'uuid': 'a', 'status': 's2'}}) + event = await _next_event(gen) + assert event.event == 'updated' and 's2' in event.data + + with pytest.raises(asyncio.TimeoutError): + await _next_event(gen, timeout=0.2) + + asyncio.run(scenario()) + + +def test_detail_stream_closes_after_delete(monkeypatch): + hub = FakeHub() + _patch(monkeypatch, {Thing: hub}) + hub.publish({b'k1': {'uuid': 'a', 'status': 'online'}}) + + async def scenario(): + gen = watch_response( + Thing, + lambda s: {k: d for k, d in s.items() if d['uuid'] == 'a'}, + _dto, + single_id='a', + ).body_iterator + assert (await _next_event(gen)).event == 'snapshot' + hub.publish({}) + assert (await _next_event(gen)).event == 'deleted' + with pytest.raises(StopAsyncIteration): + await _next_event(gen) + assert hub.subs == [] + + asyncio.run(scenario()) + + +def test_ancestor_deletion_closes_stream(monkeypatch): + hub, parent_hub = FakeHub(), FakeHub() + _patch(monkeypatch, {Thing: hub, Parent: parent_hub}) + hub.publish({b'k1': {'uuid': 'a'}}) + parent_hub.publish({b'p1': {'uuid': 'p'}}) + + async def scenario(): + gen = watch_response( + Thing, lambda s: s, _dto, ancestors=[(Parent, 'p')], + ).body_iterator + assert (await _next_event(gen)).event == 'snapshot' + parent_hub.publish({}) + with pytest.raises(StopAsyncIteration): + await _next_event(gen) + assert hub.subs == [] and parent_hub.subs == [] + + asyncio.run(scenario()) + + +def test_hub_failure_emits_error_and_closes(monkeypatch): + hub = FakeHub() + _patch(monkeypatch, {Thing: hub}) + hub.publish({b'k1': {'uuid': 'a'}}) + + async def scenario(): + gen = watch_response(Thing, lambda s: s, _dto).body_iterator + assert (await _next_event(gen)).event == 'snapshot' + hub.fail() + event = await _next_event(gen) + assert event.event == 'error' + assert 'backend unavailable' in event.data + with pytest.raises(StopAsyncIteration): + await _next_event(gen) + + asyncio.run(scenario()) + + +def test_kv_store_unavailable_yields_503(monkeypatch): + from fastapi import HTTPException + monkeypatch.setattr(_watch, 'DBController', lambda: SimpleNamespace(kv_store=None)) + with pytest.raises(HTTPException) as exc_info: + watch_response(Thing, lambda s: s, _dto) + assert exc_info.value.status_code == 503 + + +# ---- device projection (importable helper) ---- + +def test_device_projection_explodes_node(): + from simplyblock_web.api.v2.cluster.storage_node.device import _make_device_projection + state = { + b'n1': {'uuid': 'node-1', 'nvme_devices': [ + {'uuid': 'dev-1', 'status': 'online'}, + {'uuid': 'dev-2', 'status': 'unavailable'}, + ]}, + b'n2': {'uuid': 'node-2', 'nvme_devices': [{'uuid': 'dev-3'}]}, + } + assert set(_make_device_projection('node-1')(state)) == {'dev-1', 'dev-2'} + assert set(_make_device_projection('node-1', 'dev-2')(state)) == {'dev-2'} + assert _make_device_projection('node-404')(state) == {} + + +# ---- secrets on the stream (ClusterDTO carries `secret`) ---- + +def _cluster_dict(secret='hunter2'): + return {'uuid': str(uuid4()), 'secret': secret, 'status': 'active'} + + +def _cluster_dto(data: dict) -> ClusterDTO: + return ClusterDTO.from_model(ClusterModel(data), None) + + +def test_stream_payload_unwraps_secret_on_wire(caplog): + """Same behavior as the plain GET: JSON wire carries plaintext.""" + with caplog.at_level(logging.DEBUG): + events, _ = _build_snapshot({b'k1': _cluster_dict()}, _cluster_dto, single=False) + [event] = events + assert 'hunter2' in event.data + assert 'hunter2' not in caplog.text + + +def test_dto_python_mode_masks_secret(): + dto = _cluster_dto(_cluster_dict()) + assert 'hunter2' not in repr(dto) + assert str(dto.model_dump()['secret']) == '**********' From ea0cd4d92293631671583612ce2816f04cd04a15 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Thu, 2 Jul 2026 14:40:08 +0200 Subject: [PATCH 2/5] fixup! Add SSE events to API v2 --- simplyblock_web/api/v2/_watch.py | 4 +- tests/integration/test_watch_counter_bumps.py | 104 +++++++++ tests/integration/test_watch_hub.py | 100 +++++++++ tests/integration/web/__init__.py | 0 tests/integration/web/test_watch_sse_e2e.py | 204 ++++++++++++++++++ tox.ini | 2 +- 6 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_watch_counter_bumps.py create mode 100644 tests/integration/test_watch_hub.py create mode 100644 tests/integration/web/__init__.py create mode 100644 tests/integration/web/test_watch_sse_e2e.py diff --git a/simplyblock_web/api/v2/_watch.py b/simplyblock_web/api/v2/_watch.py index e03edcd58..0406fe814 100644 --- a/simplyblock_web/api/v2/_watch.py +++ b/simplyblock_web/api/v2/_watch.py @@ -219,7 +219,9 @@ def _run(self, stop: threading.Event, wake: threading.Event) -> None: try: watch.cancel() except Exception: - pass + # Best-effort cleanup: cancellation failures should not + # mask the main loop error/retry behavior. + logger.debug('Ignoring watch cancellation failure for %s', self._name, exc_info=True) _hubs: Dict[type, EntityWatchHub] = {} diff --git a/tests/integration/test_watch_counter_bumps.py b/tests/integration/test_watch_counter_bumps.py new file mode 100644 index 000000000..e36ebbf95 --- /dev/null +++ b/tests/integration/test_watch_counter_bumps.py @@ -0,0 +1,104 @@ +# coding=utf-8 +"""Every entity write path bumps the per-class change counter — real FDB.""" + +from uuid import uuid4 + +from simplyblock_core import watches +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.pool import Pool +from simplyblock_core.models.stats import LVolStatObject +from simplyblock_core.models.storage_node import StorageNode + + +def _counter(db, model_cls): + raw = db.kv_store.get(watches.watch_counter_key(model_cls)) + return watches.unpack_counter(bytes(raw)) if raw is not None else 0 + + +def _make_pool(cluster_id=None): + return Pool({ + 'uuid': str(uuid4()), + 'cluster_id': cluster_id or str(uuid4()), + 'pool_name': 'watch-test-pool', + 'status': Pool.STATUS_ACTIVE, + }) + + +def test_write_to_db_bumps_counter_by_one(): + db = DBController() + pool = _make_pool() + before = _counter(db, Pool) + pool.write_to_db(db.kv_store) + assert _counter(db, Pool) == before + 1 + assert db.get_pool_by_id(pool.get_id()).get_id() == pool.get_id() + + +def test_remove_bumps_counter(): + db = DBController() + pool = _make_pool() + pool.write_to_db(db.kv_store) + before = _counter(db, Pool) + pool.remove(db.kv_store) + assert _counter(db, Pool) == before + 1 + + +def test_atomic_update_bumps_counter(): + db = DBController() + node = StorageNode({ + 'uuid': str(uuid4()), + 'cluster_id': str(uuid4()), + 'status': StorageNode.STATUS_ONLINE, + }) + node.write_to_db(db.kv_store) + before = _counter(db, StorageNode) + + def mutate(fresh): + fresh.status = StorageNode.STATUS_SUSPENDED + + updated = db.atomic_update(node, mutate) + assert updated.status == StorageNode.STATUS_SUSPENDED + assert _counter(db, StorageNode) == before + 1 + + +def test_try_set_node_restarting_bumps_counter(monkeypatch): + from simplyblock_core import distr_controller + monkeypatch.setattr(distr_controller, 'send_node_status_event', lambda *a, **kw: None) + + db = DBController() + cluster_id = str(uuid4()) + node = StorageNode({ + 'uuid': str(uuid4()), + 'cluster_id': cluster_id, + 'status': StorageNode.STATUS_ONLINE, + }) + node.write_to_db(db.kv_store) + before = _counter(db, StorageNode) + + acquired, reason = db.try_set_node_restarting(cluster_id, node.get_id()) + assert acquired, reason + assert _counter(db, StorageNode) == before + 1 + + +def test_counter_keys_invisible_to_entity_reads(): + db = DBController() + pool = _make_pool() + pool.write_to_db(db.kv_store) + # A counter key inside the entity prefix would blow up the JSON parse in + # read_from_db; parsing the whole range cleanly is the collision guard. + pools = Pool().read_from_db(db.kv_store) + assert pool.get_id() in {p.get_id() for p in pools} + entity_keys = [bytes(k) for k, _ in db.kv_store.get_range_startswith(b'object/Pool/')] + assert all(b'watch_seq' not in key for key in entity_keys) + + +def test_unwatched_write_creates_no_counter(): + db = DBController() + before = _counter(db, LVolStatObject) + stat = LVolStatObject({ + 'uuid': str(uuid4()), + 'cluster_id': str(uuid4()), + 'pool_id': str(uuid4()), + 'date': 1, + }) + stat.write_to_db(db.kv_store) + assert _counter(db, LVolStatObject) == before diff --git a/tests/integration/test_watch_hub.py b/tests/integration/test_watch_hub.py new file mode 100644 index 000000000..06ab76afa --- /dev/null +++ b/tests/integration/test_watch_hub.py @@ -0,0 +1,100 @@ +# coding=utf-8 +"""EntityWatchHub against real FDB: watch firing, reconcile poll, teardown.""" + +import asyncio +import json +import time +from uuid import uuid4 + +import pytest + +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.pool import Pool +from simplyblock_web.api.v2 import _watch +from simplyblock_web.api.v2._watch import _UNSET, EntityWatchHub + + +def _pool_dict(pool_id=None): + return { + 'uuid': pool_id or str(uuid4()), + 'cluster_id': str(uuid4()), + 'pool_name': 'hub-test-pool', + 'status': Pool.STATUS_ACTIVE, + } + + +async def _await_publication(sub, notify, timeout=10.0): + """Wait until the subscription has a fresh state and return it.""" + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + assert remaining > 0, 'no publication within timeout' + await asyncio.wait_for(notify.wait(), remaining) + notify.clear() + state = sub.take() + if state is not _UNSET: + return state + + +@pytest.mark.timeout(60) +def test_hub_publishes_writes_and_stops_on_unsubscribe(): + db = DBController() + + async def scenario(): + loop = asyncio.get_running_loop() + notify = asyncio.Event() + hub = EntityWatchHub(Pool) + sub = hub.subscribe(loop, notify) + try: + # Initial publication (whatever the range currently holds). + await _await_publication(sub, notify) + + pool = Pool(_pool_dict()) + started = time.monotonic() + await asyncio.to_thread(pool.write_to_db, db.kv_store) + state = await _await_publication(sub, notify, timeout=5.0) + assert time.monotonic() - started < 5.0 + assert any(d.get('uuid') == pool.get_id() for d in state.values()) + + await asyncio.to_thread(pool.remove, db.kv_store) + state = await _await_publication(sub, notify, timeout=5.0) + assert not any(d.get('uuid') == pool.get_id() for d in state.values()) + finally: + hub.unsubscribe(sub) + + thread = hub._thread + deadline = time.monotonic() + 10 + while thread.is_alive() and time.monotonic() < deadline: + await asyncio.sleep(0.1) + assert not thread.is_alive() + + asyncio.run(scenario()) + + +@pytest.mark.timeout(60) +def test_reconcile_catches_writes_without_counter_bump(monkeypatch): + """Old-version writers don't bump counters; the reconcile poll must + still pick their writes up.""" + monkeypatch.setattr(_watch, 'WATCH_RECONCILE_SEC', 0.5) + db = DBController() + + async def scenario(): + loop = asyncio.get_running_loop() + notify = asyncio.Event() + hub = EntityWatchHub(Pool) + sub = hub.subscribe(loop, notify) + try: + await _await_publication(sub, notify) + + # Simulate a pre-upgrade writer: raw set, no counter bump. + legacy = _pool_dict() + key = f'object/Pool/{legacy["uuid"]}'.encode() + await asyncio.to_thread( + db.kv_store.set, key, json.dumps(legacy).encode()) + + state = await _await_publication(sub, notify, timeout=10.0) + assert any(d.get('uuid') == legacy['uuid'] for d in state.values()) + finally: + hub.unsubscribe(sub) + + asyncio.run(scenario()) diff --git a/tests/integration/web/__init__.py b/tests/integration/web/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/web/test_watch_sse_e2e.py b/tests/integration/web/test_watch_sse_e2e.py new file mode 100644 index 000000000..994c19018 --- /dev/null +++ b/tests/integration/web/test_watch_sse_e2e.py @@ -0,0 +1,204 @@ +# coding=utf-8 +"""End-to-end SSE watch: real app in a real uvicorn server on a real FDB. + +Runs the actual ``simplyblock_web.app:app`` (including AccessLogMiddleware — +the BaseHTTPMiddleware whose interaction with streaming responses this pins) +in a background uvicorn thread and consumes the stream over a real socket +with ``requests``, so client-disconnect semantics are exercised for real. +""" + +import json +import threading +import time +from uuid import uuid4 + +import pytest +import requests +import uvicorn + +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.pool import Pool +from simplyblock_web.api.v2._watch import get_hub + + +@pytest.fixture(scope='module') +def base_url(): + from simplyblock_web.app import app + from simplyblock_web.api.v2._auth import verify_api_token + + app.dependency_overrides[verify_api_token] = lambda: None + config = uvicorn.Config(app, host='127.0.0.1', port=0, log_level='warning') + server = uvicorn.Server(config) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + deadline = time.monotonic() + 15 + while not server.started: + assert time.monotonic() < deadline, 'uvicorn did not start' + time.sleep(0.05) + port = server.servers[0].sockets[0].getsockname()[1] + yield f'http://127.0.0.1:{port}' + server.should_exit = True + thread.join(timeout=10) + app.dependency_overrides.clear() + + +class SSEReader: + """Minimal SSE parser over a requests streaming response.""" + + def __init__(self, response): + # requests' iter_lines mangles \r\n-terminated SSE lines (spurious + # empty lines that are indistinguishable from event separators), so + # buffer raw content and split on \n ourselves. + self._content = response.iter_content(chunk_size=None, decode_unicode=True) + self._buffer = '' + + def _readline(self): + while '\n' not in self._buffer: + try: + self._buffer += next(self._content) + except StopIteration: + return None + line, self._buffer = self._buffer.split('\n', 1) + return line.rstrip('\r') + + def next_event(self): + """Return (event, data) for the next event, or (None, None) on EOF.""" + name, data = None, [] + while True: + line = self._readline() + if line is None: + return None, None + if line == '': + if name is not None or data: + return name, '\n'.join(data) + continue + if line.startswith(':'): + continue + field, _, value = line.partition(':') + value = value.removeprefix(' ') + if field == 'event': + name = value + elif field == 'data': + data.append(value) + + +def _seed_cluster(db): + cluster = Cluster({'uuid': str(uuid4()), 'status': Cluster.STATUS_ACTIVE}) + cluster.write_to_db(db.kv_store) + return cluster + + +def _seed_pool(db, cluster): + pool = Pool({ + 'uuid': str(uuid4()), + 'cluster_id': cluster.get_id(), + 'pool_name': f'pool-{uuid4().hex[:8]}', + 'status': Pool.STATUS_ACTIVE, + }) + pool.write_to_db(db.kv_store) + return pool + + +def _seed_lvol(db, pool, name): + lvol = LVol({ + 'uuid': str(uuid4()), + 'pool_uuid': pool.get_id(), + 'pool_name': pool.pool_name, + 'lvol_name': name, + 'node_id': str(uuid4()), + 'status': LVol.STATUS_ONLINE, + 'size': 1024, + }) + lvol.write_to_db(db.kv_store) + return lvol + + +def _volumes_watch_url(base_url, cluster, pool): + return (f'{base_url}/api/v2/clusters/{cluster.get_id()}' + f'/storage-pools/{pool.get_id()}/volumes/?watch=true') + + +@pytest.mark.timeout(120) +def test_volume_watch_stream_flow(base_url): + db = DBController() + cluster = _seed_cluster(db) + pool = _seed_pool(db, cluster) + other_pool = _seed_pool(db, cluster) + lvol = _seed_lvol(db, pool, 'watched-1') + + with requests.get(_volumes_watch_url(base_url, cluster, pool), + stream=True, timeout=(5, 30)) as response: + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + reader = SSEReader(response) + + name, data = reader.next_event() + assert name == 'snapshot' + snapshot = json.loads(data) + assert [v['id'] for v in snapshot] == [lvol.get_id()] + + # A volume in another pool must not produce an event; the next event + # observed has to be the update of the watched volume. + _seed_lvol(db, other_pool, 'other-pool-volume') + lvol.status = LVol.STATUS_OFFLINE + lvol.write_to_db(db.kv_store) + name, data = reader.next_event() + assert name == 'updated' + assert json.loads(data)['id'] == lvol.get_id() + assert json.loads(data)['status'] == LVol.STATUS_OFFLINE + + second = _seed_lvol(db, pool, 'watched-2') + name, data = reader.next_event() + assert name == 'created' + assert json.loads(data)['id'] == second.get_id() + + lvol.remove(db.kv_store) + name, data = reader.next_event() + assert name == 'deleted' + assert json.loads(data)['id'] == lvol.get_id() + + +@pytest.mark.timeout(120) +def test_parent_pool_deletion_closes_stream(base_url): + db = DBController() + cluster = _seed_cluster(db) + pool = _seed_pool(db, cluster) + _seed_lvol(db, pool, 'orphan-to-be') + + with requests.get(_volumes_watch_url(base_url, cluster, pool), + stream=True, timeout=(5, 30)) as response: + reader = SSEReader(response) + name, _ = reader.next_event() + assert name == 'snapshot' + + pool.remove(db.kv_store) + name, data = reader.next_event() + assert (name, data) == (None, None) # stream closed + + # Reconnect gets the endpoint's regular 404. + response = requests.get(_volumes_watch_url(base_url, cluster, pool), timeout=5) + assert response.status_code == 404 + + +@pytest.mark.timeout(120) +def test_client_disconnect_unsubscribes(base_url): + db = DBController() + cluster = _seed_cluster(db) + pool = _seed_pool(db, cluster) + _seed_lvol(db, pool, 'disconnect-test') + + response = requests.get(_volumes_watch_url(base_url, cluster, pool), + stream=True, timeout=(5, 30)) + reader = SSEReader(response) + name, _ = reader.next_event() + assert name == 'snapshot' + assert get_hub(LVol).subscriber_count() > 0 + + response.close() + + deadline = time.monotonic() + 20 + while get_hub(LVol).subscriber_count() > 0 and time.monotonic() < deadline: + time.sleep(0.2) + assert get_hub(LVol).subscriber_count() == 0 diff --git a/tox.ini b/tox.ini index 9d03e70a6..48b55a310 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = unit, integration, lint, types +envlist = lint, types, unit, integration skipsdist = true requires = tox-uv>=1 From b6fa9614d570dc00b90cb12c1cabd6ec89da6046 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 3 Jul 2026 10:38:04 +0200 Subject: [PATCH 3/5] fixup! Add SSE events to API v2 --- simplyblock_web/api/v2/_watch.py | 39 ++++++++++++++++----- tests/integration/web/test_watch_sse_e2e.py | 13 ++++++- tests/unit/web/api/v2/test_watch_stream.py | 35 ++++++++++++------ 3 files changed, 68 insertions(+), 19 deletions(-) diff --git a/simplyblock_web/api/v2/_watch.py b/simplyblock_web/api/v2/_watch.py index 0406fe814..829a99d0f 100644 --- a/simplyblock_web/api/v2/_watch.py +++ b/simplyblock_web/api/v2/_watch.py @@ -72,9 +72,11 @@ 'Stream state changes as Server-Sent Events instead of returning a plain ' 'response: a `snapshot` event with the current state first, then ' '`created`/`updated`/`deleted` events carrying the full resource ' - 'representation. Streams do not support resume; reconnecting clients ' - 'receive a fresh snapshot. Changes written by pre-upgrade components may ' - 'take up to 30 seconds to appear.' + 'representation. A `deleted` event carries the resource\'s final state ' + 'when it is still retrievable (e.g. a volume whose status became ' + '`deleted`), or an empty object once it is gone entirely. Streams do not ' + 'support resume; reconnecting clients receive a fresh snapshot. Changes ' + 'written by pre-upgrade components may take up to 30 seconds to appear.' ) # ``watch: WatchParam = False`` on a route adds the documented query flag. @@ -250,7 +252,7 @@ def _build_snapshot( def _build_diff( - prev: RawState, new: RawState, cache: Dict[Any, str], + prev: RawState, new: RawState, new_full: RawState, cache: Dict[Any, str], build_dto: DTOBuilder, single: bool, ) -> Tuple[List[ServerSentEvent], Dict[Any, str]]: events: List[ServerSentEvent] = [] @@ -268,8 +270,27 @@ def _build_diff( else: new_cache[key] = cache[key] for key in prev: - if key not in new: - events.append(ServerSentEvent(event='deleted', data=cache.get(key, '{}'))) + if key in new: + continue + # The entity left the projected set. If its key is still retrievable + # in the unprojected state (e.g. an LVol whose status flipped to + # 'deleted', which the list filter excludes), the event carries the + # DTO of that *current* state — not the stale last-projected one. + # A key absent from the unprojected state is physically gone, so + # there is no representation left to return. The lookup runs in the + # raw keyspace: projections that transform keys (exploded devices) + # resolve to "gone", which is correct — a device that left its + # node's list has no DB representation either. + current = new_full.get(key) + if current is not None: + try: + data = build_dto(current).model_dump_json() + except Exception: + logger.exception('Failed to build DTO for deleted entity %s', key) + data = cache.get(key, '{}') + else: + data = '{}' + events.append(ServerSentEvent(event='deleted', data=data)) return events, new_cache @@ -284,7 +305,9 @@ def watch_response( ``project`` must apply the same filter as the corresponding list endpoint; the diff runs on its output, so entities entering/leaving the filtered set - emit ``created``/``deleted``. ``single_id`` switches to detail semantics + emit ``created``/``deleted``. ``deleted`` carries the entity's current + (unprojected) state while its key is still retrievable, else ``{}`` — + see ``_build_diff``. ``single_id`` switches to detail semantics (single-DTO snapshot, close after ``deleted``). ``ancestors`` are ``(model_cls, id)`` pairs from the route's dependency chain: the stream closes when any of them disappears, so a reconnect gets the endpoint's @@ -342,7 +365,7 @@ async def event_stream() -> Any: _build_snapshot, filtered, build_dto, single) else: events, dto_cache = await run_in_threadpool( - _build_diff, prev, filtered, dto_cache, build_dto, single) + _build_diff, prev, filtered, state, dto_cache, build_dto, single) prev = filtered for event in events: yield event diff --git a/tests/integration/web/test_watch_sse_e2e.py b/tests/integration/web/test_watch_sse_e2e.py index 994c19018..e3bc34eb7 100644 --- a/tests/integration/web/test_watch_sse_e2e.py +++ b/tests/integration/web/test_watch_sse_e2e.py @@ -154,10 +154,21 @@ def test_volume_watch_stream_flow(base_url): assert name == 'created' assert json.loads(data)['id'] == second.get_id() + # Soft delete: the entity leaves the filtered set but its key is + # still retrievable — the event carries the final representation. + second.status = LVol.STATUS_DELETED + second.write_to_db(db.kv_store) + name, data = reader.next_event() + assert name == 'deleted' + payload = json.loads(data) + assert payload['id'] == second.get_id() + assert payload['status'] == LVol.STATUS_DELETED + + # Physical removal: no representation left to return. lvol.remove(db.kv_store) name, data = reader.next_event() assert name == 'deleted' - assert json.loads(data)['id'] == lvol.get_id() + assert json.loads(data) == {} @pytest.mark.timeout(120) diff --git a/tests/unit/web/api/v2/test_watch_stream.py b/tests/unit/web/api/v2/test_watch_stream.py index c78f857e0..72e122d75 100644 --- a/tests/unit/web/api/v2/test_watch_stream.py +++ b/tests/unit/web/api/v2/test_watch_stream.py @@ -48,12 +48,12 @@ def test_diff_created_updated_deleted(): prev = {b'k1': {'uuid': 'a', 'status': 'x'}, b'k2': {'uuid': 'b', 'status': 'x'}} _, cache = _build_snapshot(prev, _dto, single=False) new = {b'k2': {'uuid': 'b', 'status': 'y'}, b'k3': {'uuid': 'c', 'status': 'x'}} - events, new_cache = _build_diff(prev, new, cache, _dto, single=False) + events, new_cache = _build_diff(prev, new, new, cache, _dto, single=False) by_name = {e.event: e.data for e in events} assert set(by_name) == {'created', 'updated', 'deleted'} assert '"c"' in by_name['created'] assert '"b"' in by_name['updated'] and '"y"' in by_name['updated'] - assert '"a"' in by_name['deleted'] # last-known DTO + assert by_name['deleted'] == '{}' # key physically gone: no representation assert set(new_cache) == {b'k2', b'k3'} @@ -68,7 +68,7 @@ def test_diff_suppresses_changes_invisible_in_dto(): prev = {b'k1': task} _, cache = _build_snapshot(prev, build, single=False) new = {b'k1': {**task, 'updated_at': '2026-01-01T00:00:05'}} - events, _ = _build_diff(prev, new, cache, build, single=False) + events, _ = _build_diff(prev, new, new, cache, build, single=False) assert events == [] @@ -85,28 +85,43 @@ def test_diff_compound_keys(): prev = {k1: t1, k2: t2} _, cache = _build_snapshot(prev, build, single=False) new = {k1: t1, k2: {**t2, 'status': 'done'}} - events, _ = _build_diff(prev, new, cache, build, single=False) + events, _ = _build_diff(prev, new, new, cache, build, single=False) [event] = events assert event.event == 'updated' assert t2['uuid'] in event.data def test_diff_single_mode_maps_created_to_updated(): - events, _ = _build_diff({}, {b'k1': {'uuid': 'a'}}, {}, _dto, single=True) + new = {b'k1': {'uuid': 'a'}} + events, _ = _build_diff({}, new, new, {}, _dto, single=True) [event] = events assert event.event == 'updated' -def test_diff_scope_leave_emits_deleted(): - """An entity leaving the projected set (filtered out) emits deleted.""" +def test_diff_scope_leave_emits_deleted_with_current_state(): + """An entity leaving the projected set but still present in the DB emits + deleted carrying its *current* representation, not the stale one.""" project = lambda state: { # noqa: E731 k: d for k, d in state.items() if d.get('status') != 'deleted'} - s1 = project({b'k1': {'uuid': 'a', 'status': 'online'}}) + full1 = {b'k1': {'uuid': 'a', 'status': 'online'}} + s1 = project(full1) _, cache = _build_snapshot(s1, _dto, single=False) - s2 = project({b'k1': {'uuid': 'a', 'status': 'deleted'}}) - events, _ = _build_diff(s1, s2, cache, _dto, single=False) + full2 = {b'k1': {'uuid': 'a', 'status': 'deleted'}} + s2 = project(full2) + events, _ = _build_diff(s1, s2, full2, cache, _dto, single=False) [event] = events assert event.event == 'deleted' + assert '"deleted"' in event.data # the final state, not the cached 'online' + assert '"online"' not in event.data + + +def test_diff_physical_removal_emits_deleted_without_representation(): + full1 = {b'k1': {'uuid': 'a', 'status': 'online'}} + _, cache = _build_snapshot(full1, _dto, single=False) + events, _ = _build_diff(full1, {}, {}, cache, _dto, single=False) + [event] = events + assert event.event == 'deleted' + assert event.data == '{}' # ---- generator behavior with a fake hub ---- From 815658968a133c14636420ab2bd4f639986d368b Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 3 Jul 2026 16:55:57 +0200 Subject: [PATCH 4/5] Clear layer separation --- AGENTS.md | 19 +- simplyblock_core/cluster_ops.py | 13 + .../controllers/device_controller.py | 34 ++ .../controllers/lvol_controller.py | 23 ++ .../controllers/pool_controller.py | 21 + .../controllers/snapshot_controller.py | 24 ++ .../controllers/tasks_controller.py | 22 + simplyblock_core/db_controller.py | 52 ++- simplyblock_core/storage_node_ops.py | 21 + simplyblock_web/api/v2/_watch.py | 383 ------------------ simplyblock_web/api/v2/cluster/__init__.py | 15 +- .../api/v2/cluster/storage_node/__init__.py | 22 +- .../api/v2/cluster/storage_node/device.py | 38 +- .../api/v2/cluster/storage_pool/__init__.py | 23 +- .../api/v2/cluster/storage_pool/snapshot.py | 28 +- .../cluster/storage_pool/volume/__init__.py | 32 +- simplyblock_web/api/v2/cluster/task.py | 27 +- tests/integration/test_watch_hub.py | 18 +- tests/integration/web/test_watch_sse_e2e.py | 2 +- tests/unit/web/api/v2/test_watch_stream.py | 337 ++++++--------- 20 files changed, 396 insertions(+), 758 deletions(-) delete mode 100644 simplyblock_web/api/v2/_watch.py diff --git a/AGENTS.md b/AGENTS.md index 29d70fba5..5e341eb13 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ mypy simplyblock_web simplyblock_cli simplyblock_core # Type check (or: tox -e ## Architecture -Three packages, one entry point: +Three packages, two front-ends (CLI and Web API) over a shared core: | Package | Role | |---------|------| @@ -34,7 +34,12 @@ Three packages, one entry point: | `simplyblock_core/` | Business logic, data models, background services, FDB access | | `simplyblock_web/` | REST API — FastAPI (v2) + Flask (v1) hybrid on a single uvicorn process | -Data flows: **CLI → Web API → Core controllers → FoundationDB**. Storage nodes are reached via JSON-RPC (`rpc_client.py`). +Two front-ends sit on top of the core and **both call it in-process** — the CLI does *not* go through the Web API: + +- **`sbctl` CLI → Core controllers → FoundationDB.** `clibase.py` imports `simplyblock_core` directly (`cluster_ops`, `storage_node_ops`, the `controllers` package, `DBController`) and instantiates `DBController()` itself. +- **Web API → Core controllers → FoundationDB.** `simplyblock_web` is a separate entry point serving remote/programmatic clients over the same core. + +Storage nodes are reached via JSON-RPC (`rpc_client.py`). ## Coding Conventions @@ -109,3 +114,13 @@ Edit only `AGENTS.md` files and `.agents/skills/` contents. Never edit the symli ### Local overrides At every level where an `AGENTS.md` exists, also check for a sibling `AGENTS.local.md`. If present, load it in addition to `AGENTS.md` — its contents extend or override the checked-in instructions. `AGENTS.local.md` is gitignored and intended for per-developer notes that should not be committed. + +## graphify + +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +Rules: +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 6c86cbf33..89f829f49 100755 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -35,6 +35,19 @@ db_controller = DBController() + +def watch_clusters(): + """Stream changes across all clusters.""" + return db_controller.watch(Cluster) + + +def watch_cluster(cluster_id): + """Stream changes for a single cluster.""" + return db_controller.watch( + Cluster, + select=lambda models: [c for c in models if c.get_id() == cluster_id], + ) + def _create_update_user(cluster_id, grafana_url, grafana_secret: SecretStr, user_secret: SecretStr, update_secret=False): session = requests.session() session.auth = ("admin", grafana_secret.get_secret_value()) diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index 9ca35de17..cbbca0e10 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -5,6 +5,7 @@ from simplyblock_core import distr_controller, utils, storage_node_ops, constants from simplyblock_core.controllers import device_events, tasks_controller from simplyblock_core.db_controller import DBController +from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.prom_client import PromClient @@ -19,6 +20,39 @@ logger = logging.getLogger() +def _explode_devices(nodes, node_id): + # Devices are embedded in their StorageNode record; expose them as per-device + # entries so the diff yields device-level events. The one place that knows + # devices live inside nodes. + for node in nodes: + if node.get_id() == node_id: + return list(node.nvme_devices) + return [] + + +def watch_devices(cluster_id, node_id): + """Stream device changes for one storage node.""" + db = DBController() + return db.watch( + StorageNode, + select=lambda nodes: _explode_devices(nodes, node_id), + ancestors=[(Cluster, cluster_id), (StorageNode, node_id)], + ) + + +def watch_device(cluster_id, node_id, device_id): + """Stream changes for a single device.""" + db = DBController() + return db.watch( + StorageNode, + select=lambda nodes: [ + device for device in _explode_devices(nodes, node_id) + if device.get_id() == device_id + ], + ancestors=[(Cluster, cluster_id), (StorageNode, node_id)], + ) + + def get_storage_node_by_jm_device(db_controller: DBController, id) -> StorageNode: try: return next( diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 9b0a643f6..aafe23f2a 100755 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -28,6 +28,29 @@ logger = utils.get_logger(__name__) +def watch_volumes(cluster_id, pool_id): + """Stream volume changes for one pool (same scope as get_lvols_by_pool_id).""" + db = DBController() + return db.watch( + LVol, + select=lambda models: db.get_lvols_by_pool_id(pool_id, source=models), + ancestors=[(Cluster, cluster_id), (Pool, pool_id)], + ) + + +def watch_volume(cluster_id, pool_id, volume_id): + """Stream changes for a single volume.""" + db = DBController() + return db.watch( + LVol, + select=lambda models: [ + lvol for lvol in db.get_lvols_by_pool_id(pool_id, source=models) + if lvol.get_id() == volume_id + ], + ancestors=[(Cluster, cluster_id), (Pool, pool_id)], + ) + + def _create_crypto_lvol(rpc_client, lvol, cluster): name = lvol.crypto_bdev base_name = f"{lvol.lvs_name}/{lvol.lvol_bdev}" diff --git a/simplyblock_core/controllers/pool_controller.py b/simplyblock_core/controllers/pool_controller.py index 181e0e70e..83f5f0f7b 100644 --- a/simplyblock_core/controllers/pool_controller.py +++ b/simplyblock_core/controllers/pool_controller.py @@ -13,12 +13,33 @@ from simplyblock_core.controllers import pool_events, lvol_controller from simplyblock_core.db_controller import DBController from simplyblock_core.kms import KMSException, create_kms_connection, pool_kek_name +from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.pool import Pool from simplyblock_core.prom_client import PromClient logger = lg.getLogger() +def watch_pools(cluster_id): + """Stream pool changes for one cluster (same scope as get_pools).""" + db = DBController() + return db.watch( + Pool, + select=lambda models: db.get_pools(cluster_id, source=models), + ancestors=[(Cluster, cluster_id)], + ) + + +def watch_pool(cluster_id, pool_id): + """Stream changes for a single pool.""" + db = DBController() + return db.watch( + Pool, + select=lambda models: [pool for pool in models if pool.get_id() == pool_id], + ancestors=[(Cluster, cluster_id)], + ) + + def _generate_string(length): return ''.join(random.SystemRandom().choice( string.ascii_letters + string.digits) for _ in range(length)) diff --git a/simplyblock_core/controllers/snapshot_controller.py b/simplyblock_core/controllers/snapshot_controller.py index a608cebec..b71cb9bf9 100755 --- a/simplyblock_core/controllers/snapshot_controller.py +++ b/simplyblock_core/controllers/snapshot_controller.py @@ -19,6 +19,7 @@ from simplyblock_core.kms._exceptions import KMSException from simplyblock_core.db_controller import DBController from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.pool import Pool from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.lvol_model import LVol @@ -27,6 +28,29 @@ logger = lg.getLogger() + +def watch_snapshots(cluster_id, pool_id): + """Stream snapshot changes for one pool (same scope as get_snapshots_by_pool_id).""" + db = DBController() + return db.watch( + SnapShot, + select=lambda models: db.get_snapshots_by_pool_id(pool_id, source=models), + ancestors=[(Cluster, cluster_id), (Pool, pool_id)], + ) + + +def watch_snapshot(cluster_id, pool_id, snapshot_id): + """Stream changes for a single snapshot.""" + db = DBController() + return db.watch( + SnapShot, + select=lambda models: [ + snap for snap in db.get_snapshots_by_pool_id(pool_id, source=models) + if snap.get_id() == snapshot_id + ], + ancestors=[(Cluster, cluster_id), (Pool, pool_id)], + ) + db_controller = DBController() diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index ece14cccb..75580b8c3 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -14,6 +14,28 @@ logger = logging.getLogger() db = db_controller.DBController() + +def watch_tasks(cluster_id): + """Stream task changes for one cluster (excludes device-migration tasks, + matching the task list endpoint).""" + return db.watch( + JobSchedule, + select=lambda models: [ + task for task in db.get_job_tasks(cluster_id, source=models) + if task.function_name != JobSchedule.FN_DEV_MIG + ], + ancestors=[(Cluster, cluster_id)], + ) + + +def watch_task(cluster_id, task_id): + """Stream changes for a single task.""" + return db.watch( + JobSchedule, + select=lambda models: [task for task in models if task.uuid == task_id], + ancestors=[(Cluster, cluster_id)], + ) + # Identity used for task leases. Hostname (not pid) so a runner that crashes # and restarts on the same host re-claims its own in-flight tasks immediately. _RUNNER_HOST = socket.gethostname() diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index 62b05d0f2..ca713f3a5 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -56,13 +56,22 @@ def __init__(self): except Exception: logger.exception("FDB initialization failed") + def watch(self, model_cls, *, select=None, ancestors=()): + """Async stream of watch.ChangeEvent batches for a watched model class. + + Thin pass-through to :func:`simplyblock_core.watch.watch` so controllers + reach the watch primitive the same way they reach every other DB op. + """ + from simplyblock_core import watch as _watch + return _watch.watch(model_cls, select=select, ancestors=ancestors) + def get_storage_nodes(self) -> List[StorageNode]: ret = StorageNode().read_from_db(self.kv_store) ret = sorted(ret, key=lambda x: x.create_dt) return ret - def get_storage_nodes_by_cluster_id(self, cluster_id: str) -> List[StorageNode]: - ret = StorageNode().read_from_db(self.kv_store) + def get_storage_nodes_by_cluster_id(self, cluster_id: str, *, source=None) -> List[StorageNode]: + ret = source if source is not None else StorageNode().read_from_db(self.kv_store) nodes = [] for n in ret: if n.cluster_id == cluster_id: @@ -101,15 +110,11 @@ def get_storage_device_by_id(self, id: str) -> NVMeDevice: return device - def get_pools(self, cluster_id: Optional[str] = None) -> List[Pool]: - pools = [] + def get_pools(self, cluster_id: Optional[str] = None, *, source=None) -> List[Pool]: + all_pools = source if source is not None else Pool().read_from_db(self.kv_store) if cluster_id: - for pool in Pool().read_from_db(self.kv_store): - if pool.cluster_id == cluster_id: - pools.append(pool) - else: - pools = Pool().read_from_db(self.kv_store) - return pools + return [pool for pool in all_pools if pool.cluster_id == cluster_id] + return all_pools def get_pool_by_id(self, id: str) -> Pool: pool = single_or_none(Pool().read_from_db(self.kv_store, id)) @@ -123,8 +128,8 @@ def get_pool_by_name(self, name: str) -> Pool: raise KeyError(f'Pool {name} not found') return pool - def get_lvols(self, cluster_id: Optional[str] = None) -> List[LVol]: - lvols = self.get_all_lvols() + def get_lvols(self, cluster_id: Optional[str] = None, *, source=None) -> List[LVol]: + lvols = source if source is not None else self.get_all_lvols() lvols = [lvol for lvol in lvols if lvol.status != LVol.STATUS_DELETED] if not cluster_id: return lvols @@ -155,9 +160,9 @@ def get_lvols_by_node_id(self, node_id: str) -> List[LVol]: lvols.append(lvol) return sorted(lvols, key=lambda x: x.create_dt) - def get_lvols_by_pool_id(self, pool_id: str) -> List[LVol]: + def get_lvols_by_pool_id(self, pool_id: str, *, source=None) -> List[LVol]: lvols = [] - for lvol in self.get_lvols(): + for lvol in self.get_lvols(source=source): if lvol.pool_uuid == pool_id: lvols.append(lvol) return sorted(lvols, key=lambda x: x.create_dt) @@ -170,9 +175,9 @@ def get_hostnames_by_pool_id(self, pool_id: str) -> List[str]: hostnames.append(lv.hostname) return hostnames - def get_snapshots(self, cluster_id: Optional[str] = None) -> List[SnapShot]: + def get_snapshots(self, cluster_id: Optional[str] = None, *, source=None) -> List[SnapShot]: start_time = time.time() - snaps = SnapShot().read_from_db(self.kv_store) + snaps = source if source is not None else SnapShot().read_from_db(self.kv_store) if cluster_id: snaps = [n for n in snaps if n.cluster_id == cluster_id] ret = sorted(snaps, key=lambda x: x.created_at) @@ -283,7 +288,9 @@ def get_device_capacity(self, device, limit=1) -> List[DeviceStatObject]: self.kv_store, id="%s/%s" % (device.cluster_id, device.get_id()), limit=limit, reverse=True) return stats - def get_clusters(self) -> List[Cluster]: + def get_clusters(self, *, source=None) -> List[Cluster]: + if source is not None: + return source return Cluster().read_from_db(self.kv_store) def get_cluster_by_id(self, cluster_id: str) -> Cluster: @@ -299,8 +306,11 @@ def get_port_stats(self, node_id: str, port_id: str, limit: int = 20) -> List[Po def get_events(self, event_id: str = " ", limit: int = 0, reverse: bool = False) -> List[EventObj]: return EventObj().read_from_db(self.kv_store, id=event_id, limit=limit, reverse=reverse) - def get_job_tasks(self, cluster_id: str, reverse: bool = True, limit: int = 0) -> List[JobSchedule]: - ret = JobSchedule().read_from_db(self.kv_store, id=cluster_id, reverse=reverse, limit=limit) + def get_job_tasks(self, cluster_id: str, reverse: bool = True, limit: int = 0, *, source=None) -> List[JobSchedule]: + if source is not None: + ret = [t for t in source if t.cluster_id == cluster_id] + else: + ret = JobSchedule().read_from_db(self.kv_store, id=cluster_id, reverse=reverse, limit=limit) return sorted(ret, key=lambda x: x.date) @@ -326,9 +336,9 @@ def get_snapshots_by_node_id(self, node_id: str) -> List[SnapShot]: ret.append(snap) return sorted(ret, key=lambda x: x.create_dt) - def get_snapshots_by_pool_id(self, pool_id: str) -> List[SnapShot]: + def get_snapshots_by_pool_id(self, pool_id: str, *, source=None) -> List[SnapShot]: ret = [] - snaps = self.get_snapshots() + snaps = self.get_snapshots(source=source) for snap in snaps: if snap.pool_uuid == pool_id: ret.append(snap) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 7d19d4b27..2af054c17 100755 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -46,6 +46,27 @@ logger = utils.get_logger(__name__) +def watch_storage_nodes(cluster_id): + """Stream storage-node changes for one cluster (same scope as + get_storage_nodes_by_cluster_id).""" + db = DBController() + return db.watch( + StorageNode, + select=lambda models: db.get_storage_nodes_by_cluster_id(cluster_id, source=models), + ancestors=[(Cluster, cluster_id)], + ) + + +def watch_storage_node(cluster_id, node_id): + """Stream changes for a single storage node.""" + db = DBController() + return db.watch( + StorageNode, + select=lambda models: [node for node in models if node.get_id() == node_id], + ancestors=[(Cluster, cluster_id)], + ) + + class LVSRestartRequiredError(Exception): """Raised when an LVS fails to recover via ``bdev_examine`` during activation-mode recreate. The node's SPDK holds partial state that diff --git a/simplyblock_web/api/v2/_watch.py b/simplyblock_web/api/v2/_watch.py deleted file mode 100644 index 829a99d0f..000000000 --- a/simplyblock_web/api/v2/_watch.py +++ /dev/null @@ -1,383 +0,0 @@ -# coding=utf-8 -"""Entity watching for the v2 API (SSE ``?watch=true``). - -Change signal: every write/remove of a watched model class atomically bumps -``watch_seq/`` in the same FDB transaction as the entity mutation -(see ``simplyblock_core.watches``). One shared :class:`EntityWatchHub` per -model class owns a single FDB watch on that counter key; when it fires, the -hub re-reads the class's entity range once and fans the parsed state out to -every subscribed stream. Scoping (e.g. volumes of one pool) happens per -subscription, via the same filter its list endpoint applies — N scoped -subscribers cost one watch and one shared range read per change. - -The hub registers the watch *before* reading the range: any write not visible -to the range read has a commit version at or after the watch's read version -and therefore fires it, so no update is lost. A reconcile timeout re-reads -the range even without a watch fire, bounding staleness for writes from -pre-upgrade components that don't bump counters. -""" - -import asyncio -import json -import logging -import threading -import time -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type - -import fdb -from fastapi import HTTPException, Query -from sse_starlette import EventSourceResponse, ServerSentEvent -from starlette.concurrency import run_in_threadpool -from typing_extensions import Annotated - -from simplyblock_core import watches -from simplyblock_core.db_controller import DBController - -logger = logging.getLogger(__name__) - -# Raw FDB key -> parsed entity dict. Keyed by the FDB key (not the uuid -# field): unique by construction and indifferent to compound hierarchical ids -# (JobSchedule keys are object/JobSchedule///). -RawState = Dict[Any, dict] - -# Filter applied per subscription; must match the corresponding list endpoint. -Projection = Callable[[RawState], RawState] - -# Builds the wire DTO from a parsed entity dict. May do blocking DB reads — -# only ever called via run_in_threadpool. -DTOBuilder = Callable[[dict], Any] - -# Upper bound on staleness for entity writes that don't bump the change -# counter (processes from before the counter was introduced). -WATCH_RECONCILE_SEC = 30.0 - -# Streams are closed after this long; bearer tokens are only validated at -# request start, so this bounds how long a revoked token keeps a live stream -# (clients re-authenticate on reconnect, like Kubernetes watch timeouts). -WATCH_MAX_LIFETIME_SEC = 3600.0 - -_FDB_RETRY_BACKOFF_SEC = (1.0, 2.0, 4.0) -SSE_RETRY_MS = 3000 -PING_SEC = 15 - -_UNSET = object() - -# OpenAPI ``responses`` snippet for routes that also serve ``?watch=true``. -WATCH_RESPONSES: Dict[Any, Any] = { - 200: {'content': {'text/event-stream': {'schema': {'type': 'string'}}}}, -} - -# Query-parameter documentation, shared by all watchable routes. -WATCH_PARAM_DESCRIPTION = ( - 'Stream state changes as Server-Sent Events instead of returning a plain ' - 'response: a `snapshot` event with the current state first, then ' - '`created`/`updated`/`deleted` events carrying the full resource ' - 'representation. A `deleted` event carries the resource\'s final state ' - 'when it is still retrievable (e.g. a volume whose status became ' - '`deleted`), or an empty object once it is gone entirely. Streams do not ' - 'support resume; reconnecting clients receive a fresh snapshot. Changes ' - 'written by pre-upgrade components may take up to 30 seconds to appear.' -) - -# ``watch: WatchParam = False`` on a route adds the documented query flag. -WatchParam = Annotated[bool, Query(description=WATCH_PARAM_DESCRIPTION)] - - -class _Subscription: - """Coalescing mailbox: written by the hub thread, read by one stream.""" - - def __init__(self, loop: asyncio.AbstractEventLoop, notify: asyncio.Event) -> None: - self._loop = loop - self._notify = notify - self._latest: Any = _UNSET - self._dirty = False - self.failed = False - - def _deliver(self, state: RawState) -> None: # event-loop thread - self._latest = state - self._dirty = True - self._notify.set() - - def _fail(self) -> None: # event-loop thread - self.failed = True - self._notify.set() - - def deliver(self, state: RawState) -> None: # hub thread - self._loop.call_soon_threadsafe(self._deliver, state) - - def fail(self) -> None: # hub thread - self._loop.call_soon_threadsafe(self._fail) - - def take(self) -> Any: - """Latest state if there is an unconsumed publication, else ``_UNSET``.""" - if not self._dirty: - return _UNSET - self._dirty = False - return self._latest - - -def _watch_tx(tr: Any, key: bytes) -> Any: - return tr.watch(key) - - -class EntityWatchHub: - """Single FDB watch + shared range re-read for one watched model class.""" - - def __init__(self, model_cls: Type[Any]) -> None: - instance = model_cls() - self._name = model_cls.__name__ - # Built from parts rather than get_db_id(): a fresh JobSchedule's - # compound get_id() is not empty, which would corrupt the prefix. - self._prefix = f'{instance.object_type}/{instance.name}/'.encode() - self._counter_key = watches.watch_counter_key(model_cls) - self._lock = threading.Lock() - self._subs: set = set() - self._cache: Any = _UNSET - self._stop = threading.Event() - self._wake = threading.Event() - self._thread: Optional[threading.Thread] = None - - def subscribe(self, loop: asyncio.AbstractEventLoop, notify: asyncio.Event) -> _Subscription: - sub = _Subscription(loop, notify) - with self._lock: - alive = self._thread is not None and self._thread.is_alive() and not self._stop.is_set() - if not alive: - self._cache = _UNSET - self._stop = threading.Event() - self._wake = threading.Event() - self._thread = threading.Thread( - target=self._run, args=(self._stop, self._wake), - daemon=True, name=f'watch-{self._name}', - ) - self._thread.start() - self._subs.add(sub) - if self._cache is not _UNSET: - # subscribe() runs on the event loop thread, so deliver directly. - sub._deliver(self._cache) - return sub - - def unsubscribe(self, sub: _Subscription) -> None: - with self._lock: - self._subs.discard(sub) - if not self._subs: - self._stop.set() - self._wake.set() - - def subscriber_count(self) -> int: - with self._lock: - return len(self._subs) - - def _publish(self, state: RawState) -> None: - with self._lock: - self._cache = state - subs = list(self._subs) - for sub in subs: - sub.deliver(state) - - def _fail_all(self) -> None: - with self._lock: - subs = list(self._subs) - for sub in subs: - sub.fail() - - def _read_range(self, kv_store: Any) -> RawState: - state: RawState = {} - for k, v in kv_store.get_range_startswith(self._prefix): - try: - state[bytes(k)] = json.loads(v) - except ValueError: - logger.warning('Skipping unparsable record %s', bytes(k)) - return state - - def _run(self, stop: threading.Event, wake: threading.Event) -> None: - kv_store = DBController().kv_store - set_watch = fdb.transactional(_watch_tx) - failures = 0 - while not stop.is_set(): - watch = None - try: - # Watch first, then read: see module docstring. - watch = set_watch(kv_store, self._counter_key) - state = self._read_range(kv_store) - failures = 0 - with self._lock: - changed = self._cache is _UNSET or state != self._cache - if changed: - self._publish(state) - wake.clear() - # Runs on the FDB network thread: must only set the event, - # never block or call back into fdb. - watch.on_ready(lambda _f: wake.set()) - wake.wait(timeout=WATCH_RECONCILE_SEC) - except Exception: - logger.exception('Watch hub for %s failed', self._name) - failures += 1 - if failures > len(_FDB_RETRY_BACKOFF_SEC): - self._fail_all() - return - stop.wait(timeout=_FDB_RETRY_BACKOFF_SEC[failures - 1]) - finally: - if watch is not None: - try: - watch.cancel() - except Exception: - # Best-effort cleanup: cancellation failures should not - # mask the main loop error/retry behavior. - logger.debug('Ignoring watch cancellation failure for %s', self._name, exc_info=True) - - -_hubs: Dict[type, EntityWatchHub] = {} -_hubs_lock = threading.Lock() - - -def get_hub(model_cls: Type[Any]) -> EntityWatchHub: - with _hubs_lock: - hub = _hubs.get(model_cls) - if hub is None: - hub = _hubs[model_cls] = EntityWatchHub(model_cls) - return hub - - -def _build_snapshot( - filtered: RawState, build_dto: DTOBuilder, single: bool, -) -> Tuple[List[ServerSentEvent], Dict[Any, str]]: - cache = {key: build_dto(entity).model_dump_json() for key, entity in filtered.items()} - if single: - if not cache: - return [], cache - data = next(iter(cache.values())) - else: - data = '[' + ','.join(cache[key] for key in sorted(cache)) + ']' - return [ServerSentEvent(event='snapshot', data=data, retry=SSE_RETRY_MS)], cache - - -def _build_diff( - prev: RawState, new: RawState, new_full: RawState, cache: Dict[Any, str], - build_dto: DTOBuilder, single: bool, -) -> Tuple[List[ServerSentEvent], Dict[Any, str]]: - events: List[ServerSentEvent] = [] - new_cache: Dict[Any, str] = {} - for key, entity in new.items(): - if key not in prev: - new_cache[key] = build_dto(entity).model_dump_json() - events.append(ServerSentEvent(event='updated' if single else 'created', data=new_cache[key])) - elif entity != prev[key]: - new_cache[key] = build_dto(entity).model_dump_json() - # Suppress changes invisible in the DTO (e.g. JobSchedule's - # updated_at lease heartbeat rewrites). - if new_cache[key] != cache.get(key): - events.append(ServerSentEvent(event='updated', data=new_cache[key])) - else: - new_cache[key] = cache[key] - for key in prev: - if key in new: - continue - # The entity left the projected set. If its key is still retrievable - # in the unprojected state (e.g. an LVol whose status flipped to - # 'deleted', which the list filter excludes), the event carries the - # DTO of that *current* state — not the stale last-projected one. - # A key absent from the unprojected state is physically gone, so - # there is no representation left to return. The lookup runs in the - # raw keyspace: projections that transform keys (exploded devices) - # resolve to "gone", which is correct — a device that left its - # node's list has no DB representation either. - current = new_full.get(key) - if current is not None: - try: - data = build_dto(current).model_dump_json() - except Exception: - logger.exception('Failed to build DTO for deleted entity %s', key) - data = cache.get(key, '{}') - else: - data = '{}' - events.append(ServerSentEvent(event='deleted', data=data)) - return events, new_cache - - -def watch_response( - model_cls: Type[Any], - project: Projection, - build_dto: DTOBuilder, - single_id: Optional[str] = None, - ancestors: Sequence[Tuple[Type[Any], str]] = (), -) -> EventSourceResponse: - """SSE stream of state changes for one watched entity class. - - ``project`` must apply the same filter as the corresponding list endpoint; - the diff runs on its output, so entities entering/leaving the filtered set - emit ``created``/``deleted``. ``deleted`` carries the entity's current - (unprojected) state while its key is still retrievable, else ``{}`` — - see ``_build_diff``. ``single_id`` switches to detail semantics - (single-DTO snapshot, close after ``deleted``). ``ancestors`` are - ``(model_cls, id)`` pairs from the route's dependency chain: the stream - closes when any of them disappears, so a reconnect gets the endpoint's - regular 404. - """ - if DBController().kv_store is None: - raise HTTPException(503, 'Database unavailable') - - single = single_id is not None - ancestor_specs = [(cls, str(ancestor_id)) for cls, ancestor_id in ancestors] - - async def event_stream() -> Any: - loop = asyncio.get_running_loop() - notify = asyncio.Event() - main_sub = get_hub(model_cls).subscribe(loop, notify) - ancestor_subs = [ - (get_hub(cls).subscribe(loop, notify), cls, ancestor_id) - for cls, ancestor_id in ancestor_specs - ] - try: - prev: Optional[RawState] = None - dto_cache: Dict[Any, str] = {} - deadline = time.monotonic() + WATCH_MAX_LIFETIME_SEC - while True: - timeout = deadline - time.monotonic() - if timeout <= 0: - return - try: - await asyncio.wait_for(notify.wait(), timeout) - except asyncio.TimeoutError: - return - notify.clear() - - if main_sub.failed or any(sub.failed for sub, _, _ in ancestor_subs): - yield ServerSentEvent(event='error', data='{"detail": "backend unavailable"}') - return - - for ancestor_sub, _, ancestor_id in ancestor_subs: - ancestor_state = ancestor_sub.take() - if ancestor_state is _UNSET: - continue - if not any(entity.get('uuid') == ancestor_id for entity in ancestor_state.values()): - return - - state = main_sub.take() - if state is _UNSET: - continue - filtered = await run_in_threadpool(project, state) - if prev is None: - if single and not filtered: - # Deleted between dependency resolution and first - # publication; close so the reconnect 404s. - return - events, dto_cache = await run_in_threadpool( - _build_snapshot, filtered, build_dto, single) - else: - events, dto_cache = await run_in_threadpool( - _build_diff, prev, filtered, state, dto_cache, build_dto, single) - prev = filtered - for event in events: - yield event - if single and not filtered: - return - finally: - get_hub(model_cls).unsubscribe(main_sub) - for ancestor_sub, cls, _ in ancestor_subs: - get_hub(cls).unsubscribe(ancestor_sub) - - return EventSourceResponse( - event_stream(), - ping=PING_SEC, - headers={'Cache-Control': 'no-store', 'X-Accel-Buffering': 'no'}, - ) diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index aa14e2d68..1ee29a6e4 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -18,7 +18,7 @@ from .storage_node import api as storage_node_api from .task import api as task_api from .._dtos import ClusterDTO -from .._watch import WATCH_RESPONSES, WatchParam, watch_response +from .._sse import WATCH_RESPONSES, WatchParam, sse_response from .. import util as util @@ -90,8 +90,7 @@ class ClusterParams(BaseModel): hashicorp_vault_settings: Optional[HashicorpVaultSettings] = None -def _cluster_dto(data: dict) -> ClusterDTO: - cluster = ClusterModel(data) +def _cluster_dto(cluster: ClusterModel) -> ClusterDTO: ret = db.get_cluster_capacity(cluster, 1) return ClusterDTO.from_model(cluster, ret[0] if ret else None) @@ -99,7 +98,7 @@ def _cluster_dto(data: dict) -> ClusterDTO: @api.get('/', name='clusters:list', response_model=List[ClusterDTO], responses=WATCH_RESPONSES) def list(watch: WatchParam = False) -> Union[List[ClusterDTO], EventSourceResponse]: if watch: - return watch_response(ClusterModel, lambda state: state, _cluster_dto) + return sse_response(cluster_ops.watch_clusters(), _cluster_dto) data = [] for cluster in db.get_clusters(): stat_obj = None @@ -141,12 +140,10 @@ def add(request: Request, parameters: ClusterParams, response_format: util.Creat @instance_api.get('/', name='clusters:detail', response_model=ClusterDTO, responses=WATCH_RESPONSES) def get(cluster: Cluster, watch: WatchParam = False) -> Union[ClusterDTO, EventSourceResponse]: if watch: - cluster_id = cluster.get_id() - return watch_response( - ClusterModel, - lambda state: {k: d for k, d in state.items() if d.get('uuid') == cluster_id}, + return sse_response( + cluster_ops.watch_cluster(cluster.get_id()), _cluster_dto, - single_id=cluster_id, + single=True, ) stat_obj = None ret = db.get_cluster_capacity(cluster, 1) diff --git a/simplyblock_web/api/v2/cluster/storage_node/__init__.py b/simplyblock_web/api/v2/cluster/storage_node/__init__.py index 278095ef1..9a209de50 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_node/__init__.py @@ -8,13 +8,12 @@ from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import tasks_controller -from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.storage_node import StorageNode as StorageNodeModel from simplyblock_core import storage_node_ops from ... import util as util from ..._dependencies import Cluster, StorageNode -from ..._watch import WATCH_RESPONSES, WatchParam, watch_response +from ..._sse import WATCH_RESPONSES, WatchParam, sse_response from .device import api as device_api from ..._dtos import StorageNodeDTO, TaskDTO @@ -23,8 +22,7 @@ db = DBController() -def _storage_node_dto(data: dict) -> StorageNodeDTO: - storage_node = StorageNodeModel(data) +def _storage_node_dto(storage_node: StorageNodeModel) -> StorageNodeDTO: ret = db.get_node_capacity(storage_node, 1) return StorageNodeDTO.from_model(storage_node, ret[0] if ret else None) @@ -32,12 +30,9 @@ def _storage_node_dto(data: dict) -> StorageNodeDTO: @api.get('/', name='clusters:storage-nodes:list', response_model=List[StorageNodeDTO], responses=WATCH_RESPONSES) def list(cluster: Cluster, watch: WatchParam = False) -> Union[List[StorageNodeDTO], EventSourceResponse]: if watch: - cluster_id = cluster.get_id() - return watch_response( - StorageNodeModel, - lambda state: {k: d for k, d in state.items() if d.get('cluster_id') == cluster_id}, + return sse_response( + storage_node_ops.watch_storage_nodes(cluster.get_id()), _storage_node_dto, - ancestors=[(ClusterModel, cluster_id)], ) data = [] for storage_node in db.get_storage_nodes_by_cluster_id(cluster.get_id()): @@ -120,13 +115,10 @@ def add(request: Request, cluster: Cluster, parameters: StorageNodeParams, respo @instance_api.get('/', name='clusters:storage-nodes:detail', response_model=StorageNodeDTO, responses=WATCH_RESPONSES) def get(cluster: Cluster, storage_node: StorageNode, watch: WatchParam = False) -> Union[StorageNodeDTO, EventSourceResponse]: if watch: - node_id = storage_node.get_id() - return watch_response( - StorageNodeModel, - lambda state: {k: d for k, d in state.items() if d.get('uuid') == node_id}, + return sse_response( + storage_node_ops.watch_storage_node(cluster.get_id(), storage_node.get_id()), _storage_node_dto, - single_id=node_id, - ancestors=[(ClusterModel, cluster.get_id())], + single=True, ) node_stat_obj = None ret = db.get_node_capacity(storage_node, 1) diff --git a/simplyblock_web/api/v2/cluster/storage_node/device.py b/simplyblock_web/api/v2/cluster/storage_node/device.py index 6bb51d4b0..08753f090 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/device.py +++ b/simplyblock_web/api/v2/cluster/storage_node/device.py @@ -5,51 +5,31 @@ from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import device_controller -from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.nvme_device import NVMeDevice -from simplyblock_core.models.storage_node import StorageNode as StorageNodeModel from ..._dependencies import Cluster, StorageNode, Device from ..._dtos import DeviceDTO -from ..._watch import RawState, WATCH_RESPONSES, WatchParam, watch_response +from ..._sse import WATCH_RESPONSES, WatchParam, sse_response api = APIRouter() db = DBController() -def _make_device_dto(storage_node_id: str) -> Callable[[dict], DeviceDTO]: - def build(data: dict) -> DeviceDTO: - device = NVMeDevice(data) +def _make_device_dto(storage_node_id: str) -> Callable[[NVMeDevice], DeviceDTO]: + def build(device: NVMeDevice) -> DeviceDTO: ret = db.get_device_stats(device, 1) return DeviceDTO.from_model(device, storage_node_id, ret[0] if ret else None) return build -def _make_device_projection(node_id: str, device_id: Optional[str] = None) -> Callable[[RawState], RawState]: - # Devices are embedded in their StorageNode record; explode them into - # per-device entries so the diff yields device-level events. - def project(state: RawState) -> RawState: - for node in state.values(): - if node.get('uuid') == node_id: - return { - device['uuid']: device - for device in node.get('nvme_devices', []) - if device_id is None or device.get('uuid') == device_id - } - return {} - return project - - @api.get('/', name='clusters:storage_nodes:devices:list', response_model=List[DeviceDTO], responses=WATCH_RESPONSES) def list(cluster: Cluster, storage_node: StorageNode, watch: WatchParam = False) -> Union[List[DeviceDTO], EventSourceResponse]: if watch: node_id = storage_node.get_id() - return watch_response( - StorageNodeModel, - _make_device_projection(node_id), + return sse_response( + device_controller.watch_devices(cluster.get_id(), node_id), _make_device_dto(node_id), - ancestors=[(ClusterModel, cluster.get_id()), (StorageNodeModel, node_id)], ) data = [] for device in storage_node.nvme_devices: @@ -68,12 +48,10 @@ def list(cluster: Cluster, storage_node: StorageNode, watch: WatchParam = False) def get(cluster: Cluster, storage_node: StorageNode, device: Device, watch: WatchParam = False) -> Union[DeviceDTO, EventSourceResponse]: if watch: node_id = storage_node.get_id() - return watch_response( - StorageNodeModel, - _make_device_projection(node_id, device.get_id()), + return sse_response( + device_controller.watch_device(cluster.get_id(), node_id, device.get_id()), _make_device_dto(node_id), - single_id=device.get_id(), - ancestors=[(ClusterModel, cluster.get_id()), (StorageNodeModel, node_id)], + single=True, ) stat_obj = None ret = db.get_device_stats(device, 1) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/__init__.py b/simplyblock_web/api/v2/cluster/storage_pool/__init__.py index afbe47435..28729b115 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/__init__.py @@ -8,12 +8,11 @@ from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import pool_controller from simplyblock_core import utils as core_utils -from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.pool import Pool as PoolModel from ... import util as util from ..._dependencies import Cluster, StoragePool -from ..._watch import WATCH_RESPONSES, WatchParam, watch_response +from ..._sse import WATCH_RESPONSES, WatchParam, sse_response from .volume import api as volume_api from .snapshot import api as snapshot_api from ..._dtos import StoragePoolDTO @@ -23,19 +22,16 @@ db = DBController() -def _pool_dto(data: dict) -> StoragePoolDTO: - return StoragePoolDTO.from_model(PoolModel(data), None) +def _pool_dto(pool: PoolModel) -> StoragePoolDTO: + return StoragePoolDTO.from_model(pool, None) @api.get('/', name='clusters:storage-pools:list', response_model=List[StoragePoolDTO], responses=WATCH_RESPONSES) def list(cluster: Cluster, watch: WatchParam = False) -> Union[List[StoragePoolDTO], EventSourceResponse]: if watch: - cluster_id = cluster.get_id() - return watch_response( - PoolModel, - lambda state: {k: d for k, d in state.items() if d.get('cluster_id') == cluster_id}, + return sse_response( + pool_controller.watch_pools(cluster.get_id()), _pool_dto, - ancestors=[(ClusterModel, cluster_id)], ) return [StoragePoolDTO.from_model(pool, None) for pool in db.get_pools(cluster.get_id())] @@ -86,13 +82,10 @@ def add(request: Request, cluster: Cluster, parameters: StoragePoolParams, respo @instance_api.get('/', name='clusters:storage-pools:detail', response_model=StoragePoolDTO, responses=WATCH_RESPONSES) def get(cluster: Cluster, pool: StoragePool, watch: WatchParam = False) -> Union[StoragePoolDTO, EventSourceResponse]: if watch: - pool_id = pool.get_id() - return watch_response( - PoolModel, - lambda state: {k: d for k, d in state.items() if d.get('uuid') == pool_id}, + return sse_response( + pool_controller.watch_pool(cluster.get_id(), pool.get_id()), _pool_dto, - single_id=pool_id, - ancestors=[(ClusterModel, cluster.get_id())], + single=True, ) stat_obj = None return StoragePoolDTO.from_model(pool, stat_obj) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/snapshot.py b/simplyblock_web/api/v2/cluster/storage_pool/snapshot.py index be7372976..11dfc0ac4 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/snapshot.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/snapshot.py @@ -5,34 +5,29 @@ from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import snapshot_controller -from simplyblock_core.models.cluster import Cluster as ClusterModel -from simplyblock_core.models.pool import Pool as PoolModel from simplyblock_core.models.snapshot import SnapShot as SnapshotModel from ..._dependencies import Cluster, StoragePool, Snapshot from ..._dtos import SnapshotDTO -from ..._watch import WATCH_RESPONSES, WatchParam, watch_response +from ..._sse import WATCH_RESPONSES, WatchParam, sse_response api = APIRouter() db = DBController() -def _make_snapshot_dto(request: Request, cluster_id: str, pool_id: str) -> Callable[[dict], SnapshotDTO]: - def build(data: dict) -> SnapshotDTO: - return SnapshotDTO.from_model(SnapshotModel(data), request, cluster_id=cluster_id, pool_id=pool_id) +def _make_snapshot_dto(request: Request, cluster_id: str, pool_id: str) -> Callable[[SnapshotModel], SnapshotDTO]: + def build(snapshot: SnapshotModel) -> SnapshotDTO: + return SnapshotDTO.from_model(snapshot, request, cluster_id=cluster_id, pool_id=pool_id) return build @api.get('/', name='clusters:storage-pools:snapshots:list', response_model=List[SnapshotDTO], responses=WATCH_RESPONSES) def list(request: Request, cluster: Cluster, pool: StoragePool, watch: WatchParam = False) -> Union[List[SnapshotDTO], EventSourceResponse]: if watch: - pool_id = pool.get_id() - return watch_response( - SnapshotModel, - lambda state: {k: d for k, d in state.items() if d.get('pool_uuid') == pool_id}, - _make_snapshot_dto(request, cluster.get_id(), pool_id), - ancestors=[(ClusterModel, cluster.get_id()), (PoolModel, pool_id)], + return sse_response( + snapshot_controller.watch_snapshots(cluster.get_id(), pool.get_id()), + _make_snapshot_dto(request, cluster.get_id(), pool.get_id()), ) return [ SnapshotDTO.from_model(snapshot, request, cluster_id=cluster.get_id(), pool_id=pool.get_id()) @@ -46,13 +41,10 @@ def list(request: Request, cluster: Cluster, pool: StoragePool, watch: WatchPara @instance_api.get('/', name='clusters:storage-pools:snapshots:detail', response_model=SnapshotDTO, responses=WATCH_RESPONSES) def get(request: Request, cluster: Cluster, pool: StoragePool, snapshot: Snapshot, watch: WatchParam = False) -> Union[SnapshotDTO, EventSourceResponse]: if watch: - snapshot_id = snapshot.get_id() - return watch_response( - SnapshotModel, - lambda state: {k: d for k, d in state.items() if d.get('uuid') == snapshot_id}, + return sse_response( + snapshot_controller.watch_snapshot(cluster.get_id(), pool.get_id(), snapshot.get_id()), _make_snapshot_dto(request, cluster.get_id(), pool.get_id()), - single_id=snapshot_id, - ancestors=[(ClusterModel, cluster.get_id()), (PoolModel, pool.get_id())], + single=True, ) return SnapshotDTO.from_model(snapshot, request, cluster_id=cluster.get_id(), pool_id=pool.get_id()) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py index ca899c0d4..e9f486e22 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py @@ -8,13 +8,11 @@ from simplyblock_core.db_controller import DBController from simplyblock_core import utils as core_utils from simplyblock_core.controllers import backup_controller, lvol_controller, snapshot_controller -from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.lvol_model import LVol -from simplyblock_core.models.pool import Pool as PoolModel from ...._dependencies import Cluster, StoragePool, Volume from ...._dtos import BackupDTO, VolumeDTO, SnapshotDTO, TaskDTO -from ...._watch import WATCH_RESPONSES, WatchParam, watch_response +from ...._sse import WATCH_RESPONSES, WatchParam, sse_response from .... import util from .migration import api as migration_api @@ -26,17 +24,10 @@ @api.get('/', name='clusters:storage-pools:volumes:list', response_model=List[VolumeDTO], responses=WATCH_RESPONSES) def list(request: Request, cluster: Cluster, pool: StoragePool, watch: WatchParam = False) -> Union[List[VolumeDTO], EventSourceResponse]: if watch: - pool_id = pool.get_id() cluster_id = cluster.get_id() - return watch_response( - LVol, - # Same filter as get_lvols_by_pool_id: pool membership, deleted excluded. - lambda state: { - k: d for k, d in state.items() - if d.get('pool_uuid') == pool_id and d.get('status') != LVol.STATUS_DELETED - }, - lambda d: VolumeDTO.from_model(LVol(d), request, cluster_id, None), - ancestors=[(ClusterModel, cluster_id), (PoolModel, pool_id)], + return sse_response( + lvol_controller.watch_volumes(cluster_id, pool.get_id()), + lambda lvol: VolumeDTO.from_model(lvol, request, cluster_id, None), ) data = [] for lvol in db.get_lvols_by_pool_id(pool.get_id()): @@ -160,23 +151,16 @@ def replicate_lvol_on_source_cluster(cluster: Cluster, pool: StoragePool, body: @instance_api.get('/', name='clusters:storage-pools:volumes:detail', response_model=VolumeDTO, responses=WATCH_RESPONSES) def get(request: Request, cluster: Cluster, pool: StoragePool, volume: Volume, watch: WatchParam = False) -> Union[VolumeDTO, EventSourceResponse]: if watch: - volume_id = volume.get_id() cluster_id = cluster.get_id() - def _volume_dto(d: dict) -> VolumeDTO: - lvol = LVol(d) + def _volume_dto(lvol: LVol) -> VolumeDTO: rep_info = lvol_controller.get_replication_info(lvol.get_id()) return VolumeDTO.from_model(lvol, request, cluster_id, None, rep_info) - return watch_response( - LVol, - lambda state: { - k: d for k, d in state.items() - if d.get('uuid') == volume_id and d.get('status') != LVol.STATUS_DELETED - }, + return sse_response( + lvol_controller.watch_volume(cluster_id, pool.get_id(), volume.get_id()), _volume_dto, - single_id=volume_id, - ancestors=[(ClusterModel, cluster_id), (PoolModel, pool.get_id())], + single=True, ) stat_obj = None rep_info = lvol_controller.get_replication_info(volume.get_id()) diff --git a/simplyblock_web/api/v2/cluster/task.py b/simplyblock_web/api/v2/cluster/task.py index d72d70f4c..d81aec481 100644 --- a/simplyblock_web/api/v2/cluster/task.py +++ b/simplyblock_web/api/v2/cluster/task.py @@ -4,34 +4,28 @@ from sse_starlette import EventSourceResponse from simplyblock_core.db_controller import DBController -from simplyblock_core.models.cluster import Cluster as ClusterModel +from simplyblock_core.controllers import tasks_controller from simplyblock_core.models.job_schedule import JobSchedule from .._dependencies import Cluster, Task from .._dtos import TaskDTO -from .._watch import WATCH_RESPONSES, WatchParam, watch_response +from .._sse import WATCH_RESPONSES, WatchParam, sse_response api = APIRouter() db = DBController() -def _task_dto(data: dict) -> TaskDTO: - return TaskDTO.from_model(JobSchedule(data)) +def _task_dto(task: JobSchedule) -> TaskDTO: + return TaskDTO.from_model(task) @api.get('/', name='clusters:tasks:list', response_model=List[TaskDTO], responses=WATCH_RESPONSES) def list(cluster: Cluster, watch: WatchParam = False) -> Union[List[TaskDTO], EventSourceResponse]: if watch: - cluster_id = cluster.get_id() - return watch_response( - JobSchedule, - lambda state: { - k: d for k, d in state.items() - if d.get('cluster_id') == cluster_id and d.get('function_name') != JobSchedule.FN_DEV_MIG - }, + return sse_response( + tasks_controller.watch_tasks(cluster.get_id()), _task_dto, - ancestors=[(ClusterModel, cluster_id)], ) cluster_tasks = db.get_job_tasks(cluster.get_id(), limit=0) data = [] @@ -48,13 +42,10 @@ def list(cluster: Cluster, watch: WatchParam = False) -> Union[List[TaskDTO], Ev @instance_api.get('/', name='clusters:tasks:detail', response_model=TaskDTO, responses=WATCH_RESPONSES) def get(cluster: Cluster, task: Task, watch: WatchParam = False) -> Union[TaskDTO, EventSourceResponse]: if watch: - task_uuid = task.uuid - return watch_response( - JobSchedule, - lambda state: {k: d for k, d in state.items() if d.get('uuid') == task_uuid}, + return sse_response( + tasks_controller.watch_task(cluster.get_id(), task.uuid), _task_dto, - single_id=task_uuid, - ancestors=[(ClusterModel, cluster.get_id())], + single=True, ) return TaskDTO.from_model(task) diff --git a/tests/integration/test_watch_hub.py b/tests/integration/test_watch_hub.py index 06ab76afa..00323c320 100644 --- a/tests/integration/test_watch_hub.py +++ b/tests/integration/test_watch_hub.py @@ -1,5 +1,5 @@ # coding=utf-8 -"""EntityWatchHub against real FDB: watch firing, reconcile poll, teardown.""" +"""WatchHub against real FDB: watch firing, reconcile poll, teardown.""" import asyncio import json @@ -10,8 +10,8 @@ from simplyblock_core.db_controller import DBController from simplyblock_core.models.pool import Pool -from simplyblock_web.api.v2 import _watch -from simplyblock_web.api.v2._watch import _UNSET, EntityWatchHub +from simplyblock_core import watch +from simplyblock_core.watch import _UNSET, WatchHub def _pool_dict(pool_id=None): @@ -43,7 +43,7 @@ def test_hub_publishes_writes_and_stops_on_unsubscribe(): async def scenario(): loop = asyncio.get_running_loop() notify = asyncio.Event() - hub = EntityWatchHub(Pool) + hub = WatchHub(Pool) sub = hub.subscribe(loop, notify) try: # Initial publication (whatever the range currently holds). @@ -54,11 +54,11 @@ async def scenario(): await asyncio.to_thread(pool.write_to_db, db.kv_store) state = await _await_publication(sub, notify, timeout=5.0) assert time.monotonic() - started < 5.0 - assert any(d.get('uuid') == pool.get_id() for d in state.values()) + assert any(m.get_id() == pool.get_id() for m in state) await asyncio.to_thread(pool.remove, db.kv_store) state = await _await_publication(sub, notify, timeout=5.0) - assert not any(d.get('uuid') == pool.get_id() for d in state.values()) + assert not any(m.get_id() == pool.get_id() for m in state) finally: hub.unsubscribe(sub) @@ -75,13 +75,13 @@ async def scenario(): def test_reconcile_catches_writes_without_counter_bump(monkeypatch): """Old-version writers don't bump counters; the reconcile poll must still pick their writes up.""" - monkeypatch.setattr(_watch, 'WATCH_RECONCILE_SEC', 0.5) + monkeypatch.setattr(watch, 'WATCH_RECONCILE_SEC', 0.5) db = DBController() async def scenario(): loop = asyncio.get_running_loop() notify = asyncio.Event() - hub = EntityWatchHub(Pool) + hub = WatchHub(Pool) sub = hub.subscribe(loop, notify) try: await _await_publication(sub, notify) @@ -93,7 +93,7 @@ async def scenario(): db.kv_store.set, key, json.dumps(legacy).encode()) state = await _await_publication(sub, notify, timeout=10.0) - assert any(d.get('uuid') == legacy['uuid'] for d in state.values()) + assert any(m.get_id() == legacy['uuid'] for m in state) finally: hub.unsubscribe(sub) diff --git a/tests/integration/web/test_watch_sse_e2e.py b/tests/integration/web/test_watch_sse_e2e.py index e3bc34eb7..f85d8fdbd 100644 --- a/tests/integration/web/test_watch_sse_e2e.py +++ b/tests/integration/web/test_watch_sse_e2e.py @@ -20,7 +20,7 @@ from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.lvol_model import LVol from simplyblock_core.models.pool import Pool -from simplyblock_web.api.v2._watch import get_hub +from simplyblock_core.watch import get_hub @pytest.fixture(scope='module') diff --git a/tests/unit/web/api/v2/test_watch_stream.py b/tests/unit/web/api/v2/test_watch_stream.py index 72e122d75..704f8d028 100644 --- a/tests/unit/web/api/v2/test_watch_stream.py +++ b/tests/unit/web/api/v2/test_watch_stream.py @@ -1,323 +1,234 @@ # coding=utf-8 -"""Watch stream builder: diffing, event ordering, ancestor close, secrets.""" +"""SSE framing (_sse.py): snapshot/diff rendering, suppression, secrets.""" import asyncio import logging -from types import SimpleNamespace from uuid import uuid4 import pytest from pydantic import BaseModel +from simplyblock_core.watch import ChangeEvent, WatchUnavailable from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.job_schedule import JobSchedule -from simplyblock_web.api.v2 import _watch +from simplyblock_web.api.v2 import _sse +from simplyblock_web.api.v2._sse import _build_events, _build_snapshot, sse_response from simplyblock_web.api.v2._dtos import ClusterDTO, TaskDTO -from simplyblock_web.api.v2._watch import _Subscription, _build_diff, _build_snapshot, watch_response + + +class FakeModel: + def __init__(self, entity_id, **fields): + self._id = entity_id + self._fields = fields + + def get_id(self): + return self._id + + def to_dict(self): + return {'uuid': self._id, **self._fields} class _DTO(BaseModel): - uuid: str + id: str status: str = '' -def _dto(entity: dict) -> _DTO: - return _DTO(uuid=entity['uuid'], status=entity.get('status', '')) +def _dto(model) -> _DTO: + return _DTO(id=model.get_id(), status=model.to_dict().get('status', '')) + + +def _created(entity_id, **f): + return ChangeEvent('created', entity_id, FakeModel(entity_id, **f)) + +def _updated(entity_id, **f): + return ChangeEvent('updated', entity_id, FakeModel(entity_id, **f)) -# ---- _build_snapshot / _build_diff (pure functions) ---- + +def _deleted(entity_id, model=None): + return ChangeEvent('deleted', entity_id, model) + + +# ---- _build_snapshot ---- def test_snapshot_collection_is_json_array(): - events, cache = _build_snapshot( - {b'k1': {'uuid': 'a'}, b'k2': {'uuid': 'b'}}, _dto, single=False) + events, cache = _build_snapshot([_created('a'), _created('b')], _dto, single=False) [event] = events assert event.event == 'snapshot' assert event.data.startswith('[') and event.data.endswith(']') assert '"a"' in event.data and '"b"' in event.data - assert set(cache) == {b'k1', b'k2'} + assert set(cache) == {'a', 'b'} def test_snapshot_single_is_bare_object(): - events, _ = _build_snapshot({b'k1': {'uuid': 'a'}}, _dto, single=True) + events, _ = _build_snapshot([_created('a')], _dto, single=True) [event] = events assert event.event == 'snapshot' assert event.data.startswith('{') -def test_diff_created_updated_deleted(): - prev = {b'k1': {'uuid': 'a', 'status': 'x'}, b'k2': {'uuid': 'b', 'status': 'x'}} - _, cache = _build_snapshot(prev, _dto, single=False) - new = {b'k2': {'uuid': 'b', 'status': 'y'}, b'k3': {'uuid': 'c', 'status': 'x'}} - events, new_cache = _build_diff(prev, new, new, cache, _dto, single=False) +def test_snapshot_single_empty_emits_nothing(): + events, cache = _build_snapshot([], _dto, single=True) + assert events == [] and cache == {} + + +# ---- _build_events ---- + +def test_events_created_updated_deleted(): + _, cache = _build_snapshot([_created('a', status='x')], _dto, single=False) + events, new_cache, deleted = _build_events( + [_updated('a', status='y'), _created('c', status='x'), _deleted('b')], + cache, _dto, single=False) by_name = {e.event: e.data for e in events} - assert set(by_name) == {'created', 'updated', 'deleted'} + assert set(by_name) == {'updated', 'created', 'deleted'} + assert '"y"' in by_name['updated'] assert '"c"' in by_name['created'] - assert '"b"' in by_name['updated'] and '"y"' in by_name['updated'] - assert by_name['deleted'] == '{}' # key physically gone: no representation - assert set(new_cache) == {b'k2', b'k3'} + assert by_name['deleted'] == '{}' # physically gone: no representation + assert deleted is True + assert set(new_cache) == {'a', 'c'} -def test_diff_suppresses_changes_invisible_in_dto(): - """Raw dict changed (lease heartbeat) but the DTO is identical -> no event.""" +def test_events_suppress_dto_invisible_update(): + """Raw model changed (lease heartbeat) but the DTO is identical -> no event.""" task = { 'uuid': str(uuid4()), 'cluster_id': str(uuid4()), 'date': 1, 'function_name': 'node_restart', 'status': 'running', 'updated_at': '2026-01-01T00:00:00', } - build = lambda d: TaskDTO.from_model(JobSchedule(d)) # noqa: E731 - prev = {b'k1': task} - _, cache = _build_snapshot(prev, build, single=False) - new = {b'k1': {**task, 'updated_at': '2026-01-01T00:00:05'}} - events, _ = _build_diff(prev, new, new, cache, build, single=False) + build = lambda m: TaskDTO.from_model(m) # noqa: E731 + js1 = JobSchedule(task) + _, cache, _ = _build_events([ChangeEvent('created', js1.get_id(), js1)], {}, build, single=False) + js2 = JobSchedule({**task, 'updated_at': '2026-01-01T00:00:05'}) + events, _, _ = _build_events([ChangeEvent('updated', js2.get_id(), js2)], cache, build, single=False) assert events == [] -def test_diff_compound_keys(): - """JobSchedule FDB keys share a cluster prefix; diff must key exactly.""" - cluster_id = str(uuid4()) - t1 = {'uuid': str(uuid4()), 'cluster_id': cluster_id, 'date': 1, - 'function_name': 'node_restart', 'status': 'running'} - t2 = {'uuid': str(uuid4()), 'cluster_id': cluster_id, 'date': 2, - 'function_name': 'node_restart', 'status': 'running'} - build = lambda d: TaskDTO.from_model(JobSchedule(d)) # noqa: E731 - k1 = f'object/JobSchedule/{cluster_id}/1/{t1["uuid"]}'.encode() - k2 = f'object/JobSchedule/{cluster_id}/2/{t2["uuid"]}'.encode() - prev = {k1: t1, k2: t2} - _, cache = _build_snapshot(prev, build, single=False) - new = {k1: t1, k2: {**t2, 'status': 'done'}} - events, _ = _build_diff(prev, new, new, cache, build, single=False) +def test_events_single_maps_created_to_updated(): + events, _, _ = _build_events([_created('a')], {}, _dto, single=True) [event] = events assert event.event == 'updated' - assert t2['uuid'] in event.data -def test_diff_single_mode_maps_created_to_updated(): - new = {b'k1': {'uuid': 'a'}} - events, _ = _build_diff({}, new, new, {}, _dto, single=True) - [event] = events - assert event.event == 'updated' - - -def test_diff_scope_leave_emits_deleted_with_current_state(): - """An entity leaving the projected set but still present in the DB emits - deleted carrying its *current* representation, not the stale one.""" - project = lambda state: { # noqa: E731 - k: d for k, d in state.items() if d.get('status') != 'deleted'} - full1 = {b'k1': {'uuid': 'a', 'status': 'online'}} - s1 = project(full1) - _, cache = _build_snapshot(s1, _dto, single=False) - full2 = {b'k1': {'uuid': 'a', 'status': 'deleted'}} - s2 = project(full2) - events, _ = _build_diff(s1, s2, full2, cache, _dto, single=False) +def test_events_deleted_carries_current_state(): + events, _, _ = _build_events( + [_deleted('a', FakeModel('a', status='deleted'))], {}, _dto, single=False) [event] = events assert event.event == 'deleted' - assert '"deleted"' in event.data # the final state, not the cached 'online' - assert '"online"' not in event.data + assert '"deleted"' in event.data -def test_diff_physical_removal_emits_deleted_without_representation(): - full1 = {b'k1': {'uuid': 'a', 'status': 'online'}} - _, cache = _build_snapshot(full1, _dto, single=False) - events, _ = _build_diff(full1, {}, {}, cache, _dto, single=False) - [event] = events - assert event.event == 'deleted' - assert event.data == '{}' +# ---- sse_response generator ---- +class FakeStream: + def __init__(self, batches): + self._it = iter(batches) + self.closed = False -# ---- generator behavior with a fake hub ---- + def __aiter__(self): + return self -class FakeHub: - def __init__(self): - self.subs = [] - self.cache = None + async def __anext__(self): + try: + return next(self._it) + except StopIteration: + raise StopAsyncIteration - def subscribe(self, loop, notify): - sub = _Subscription(loop, notify) - self.subs.append(sub) - if self.cache is not None: - sub._deliver(self.cache) - return sub + async def aclose(self): + self.closed = True - def unsubscribe(self, sub): - self.subs.remove(sub) - - def publish(self, state): - self.cache = state - for sub in list(self.subs): - sub._deliver(state) - - def fail(self): - for sub in list(self.subs): - sub._fail() - - -class Thing: - pass +class FailingStream: + def __init__(self): + self.n = 0 -class Parent: - pass + def __aiter__(self): + return self + async def __anext__(self): + self.n += 1 + if self.n == 1: + return [_created('a')] + raise WatchUnavailable() -def _patch(monkeypatch, hubs): - monkeypatch.setattr(_watch, 'DBController', lambda: SimpleNamespace(kv_store=object())) - monkeypatch.setattr(_watch, 'get_hub', lambda cls: hubs[cls]) + async def aclose(self): + pass -async def _next_event(gen, timeout=2.0): +async def _event(gen, timeout=2.0): return await asyncio.wait_for(gen.__anext__(), timeout) def test_stream_snapshot_update_delete(monkeypatch): - hub = FakeHub() - _patch(monkeypatch, {Thing: hub}) - hub.publish({b'k1': {'uuid': 'a', 'status': 'online'}}) + monkeypatch.setattr(_sse, 'backend_available', lambda: True) async def scenario(): - response = watch_response(Thing, lambda s: s, _dto) - gen = response.body_iterator - event = await _next_event(gen) - assert event.event == 'snapshot' - - hub.publish({b'k1': {'uuid': 'a', 'status': 'offline'}}) - event = await _next_event(gen) + stream = FakeStream([[_created('a', status='online')], + [_updated('a', status='offline')], + [_deleted('a')]]) + gen = sse_response(stream, _dto).body_iterator + assert (await _event(gen)).event == 'snapshot' + event = await _event(gen) assert event.event == 'updated' and 'offline' in event.data - - hub.publish({}) - event = await _next_event(gen) - assert event.event == 'deleted' - - await gen.aclose() - assert hub.subs == [] - - asyncio.run(scenario()) - - -def test_stream_coalesces_bursts(monkeypatch): - hub = FakeHub() - _patch(monkeypatch, {Thing: hub}) - hub.publish({b'k1': {'uuid': 'a', 'status': 's0'}}) - - async def scenario(): - gen = watch_response(Thing, lambda s: s, _dto).body_iterator - assert (await _next_event(gen)).event == 'snapshot' - - # Two publications without yielding to the generator: only the - # latest state is diffed. - hub.publish({b'k1': {'uuid': 'a', 'status': 's1'}}) - hub.publish({b'k1': {'uuid': 'a', 'status': 's2'}}) - event = await _next_event(gen) - assert event.event == 'updated' and 's2' in event.data - - with pytest.raises(asyncio.TimeoutError): - await _next_event(gen, timeout=0.2) - - asyncio.run(scenario()) - - -def test_detail_stream_closes_after_delete(monkeypatch): - hub = FakeHub() - _patch(monkeypatch, {Thing: hub}) - hub.publish({b'k1': {'uuid': 'a', 'status': 'online'}}) - - async def scenario(): - gen = watch_response( - Thing, - lambda s: {k: d for k, d in s.items() if d['uuid'] == 'a'}, - _dto, - single_id='a', - ).body_iterator - assert (await _next_event(gen)).event == 'snapshot' - hub.publish({}) - assert (await _next_event(gen)).event == 'deleted' + assert (await _event(gen)).event == 'deleted' with pytest.raises(StopAsyncIteration): - await _next_event(gen) - assert hub.subs == [] + await _event(gen) + assert stream.closed asyncio.run(scenario()) -def test_ancestor_deletion_closes_stream(monkeypatch): - hub, parent_hub = FakeHub(), FakeHub() - _patch(monkeypatch, {Thing: hub, Parent: parent_hub}) - hub.publish({b'k1': {'uuid': 'a'}}) - parent_hub.publish({b'p1': {'uuid': 'p'}}) +def test_detail_stream_closes_after_delete(monkeypatch): + monkeypatch.setattr(_sse, 'backend_available', lambda: True) async def scenario(): - gen = watch_response( - Thing, lambda s: s, _dto, ancestors=[(Parent, 'p')], - ).body_iterator - assert (await _next_event(gen)).event == 'snapshot' - parent_hub.publish({}) + stream = FakeStream([[_created('a')], [_deleted('a')]]) + gen = sse_response(stream, _dto, single=True).body_iterator + assert (await _event(gen)).event == 'snapshot' + assert (await _event(gen)).event == 'deleted' with pytest.raises(StopAsyncIteration): - await _next_event(gen) - assert hub.subs == [] and parent_hub.subs == [] + await _event(gen) asyncio.run(scenario()) -def test_hub_failure_emits_error_and_closes(monkeypatch): - hub = FakeHub() - _patch(monkeypatch, {Thing: hub}) - hub.publish({b'k1': {'uuid': 'a'}}) +def test_stream_failure_emits_error_and_closes(monkeypatch): + monkeypatch.setattr(_sse, 'backend_available', lambda: True) async def scenario(): - gen = watch_response(Thing, lambda s: s, _dto).body_iterator - assert (await _next_event(gen)).event == 'snapshot' - hub.fail() - event = await _next_event(gen) - assert event.event == 'error' - assert 'backend unavailable' in event.data + gen = sse_response(FailingStream(), _dto).body_iterator + assert (await _event(gen)).event == 'snapshot' + event = await _event(gen) + assert event.event == 'error' and 'backend unavailable' in event.data with pytest.raises(StopAsyncIteration): - await _next_event(gen) + await _event(gen) asyncio.run(scenario()) def test_kv_store_unavailable_yields_503(monkeypatch): from fastapi import HTTPException - monkeypatch.setattr(_watch, 'DBController', lambda: SimpleNamespace(kv_store=None)) + monkeypatch.setattr(_sse, 'backend_available', lambda: False) with pytest.raises(HTTPException) as exc_info: - watch_response(Thing, lambda s: s, _dto) + sse_response(FakeStream([]), _dto) assert exc_info.value.status_code == 503 -# ---- device projection (importable helper) ---- - -def test_device_projection_explodes_node(): - from simplyblock_web.api.v2.cluster.storage_node.device import _make_device_projection - state = { - b'n1': {'uuid': 'node-1', 'nvme_devices': [ - {'uuid': 'dev-1', 'status': 'online'}, - {'uuid': 'dev-2', 'status': 'unavailable'}, - ]}, - b'n2': {'uuid': 'node-2', 'nvme_devices': [{'uuid': 'dev-3'}]}, - } - assert set(_make_device_projection('node-1')(state)) == {'dev-1', 'dev-2'} - assert set(_make_device_projection('node-1', 'dev-2')(state)) == {'dev-2'} - assert _make_device_projection('node-404')(state) == {} - - # ---- secrets on the stream (ClusterDTO carries `secret`) ---- -def _cluster_dict(secret='hunter2'): - return {'uuid': str(uuid4()), 'secret': secret, 'status': 'active'} - - -def _cluster_dto(data: dict) -> ClusterDTO: - return ClusterDTO.from_model(ClusterModel(data), None) - - def test_stream_payload_unwraps_secret_on_wire(caplog): """Same behavior as the plain GET: JSON wire carries plaintext.""" + cluster = ClusterModel({'uuid': str(uuid4()), 'secret': 'hunter2', 'status': 'active'}) + build = lambda m: ClusterDTO.from_model(m, None) # noqa: E731 with caplog.at_level(logging.DEBUG): - events, _ = _build_snapshot({b'k1': _cluster_dict()}, _cluster_dto, single=False) + events, _ = _build_snapshot( + [ChangeEvent('created', cluster.get_id(), cluster)], build, single=False) [event] = events assert 'hunter2' in event.data assert 'hunter2' not in caplog.text def test_dto_python_mode_masks_secret(): - dto = _cluster_dto(_cluster_dict()) + cluster = ClusterModel({'uuid': str(uuid4()), 'secret': 'hunter2', 'status': 'active'}) + dto = ClusterDTO.from_model(cluster, None) assert 'hunter2' not in repr(dto) assert str(dto.model_dump()['secret']) == '**********' From f96a7238bcd32b38dd892f69079265ff51125b89 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Tue, 7 Jul 2026 12:06:00 +0200 Subject: [PATCH 5/5] fixup! Clear layer separation --- simplyblock_core/watch.py | 336 ++++++++++++++++++ simplyblock_web/api/v2/_sse.py | 162 +++++++++ tests/unit/test_watch_core.py | 197 ++++++++++ tests/unit/web/api/v2/conftest.py | 50 +++ .../unit/web/api/v2/test_cluster_endpoints.py | 25 ++ .../unit/web/api/v2/test_device_endpoints.py | 26 ++ .../web/api/v2/test_snapshot_endpoints.py | 26 ++ .../web/api/v2/test_storage_node_endpoints.py | 25 ++ .../web/api/v2/test_storage_pool_endpoints.py | 25 ++ tests/unit/web/api/v2/test_task_endpoints.py | 25 ++ .../unit/web/api/v2/test_volume_endpoints.py | 25 ++ 11 files changed, 922 insertions(+) create mode 100644 simplyblock_core/watch.py create mode 100644 simplyblock_web/api/v2/_sse.py create mode 100644 tests/unit/test_watch_core.py diff --git a/simplyblock_core/watch.py b/simplyblock_core/watch.py new file mode 100644 index 000000000..539702b99 --- /dev/null +++ b/simplyblock_core/watch.py @@ -0,0 +1,336 @@ +# coding=utf-8 +"""Generic entity-watch primitive for streaming state changes. + +Change signal: every write/remove of a watched model class atomically bumps +``watch_seq/`` in the same FDB transaction as the entity mutation +(see :mod:`simplyblock_core.watches`). One shared :class:`WatchHub` per model +class owns a single FDB watch on that counter key; when it fires, the hub +re-reads the class's entity range once and fans the parsed models out to every +subscriber. Scoping (e.g. volumes of one pool) happens per subscription via a +``select`` callable, so N scoped subscribers cost one watch and one shared range +read per change. + +The hub registers the watch *before* reading the range: any write not visible to +the range read has a commit version at or after the watch's read version and +therefore fires it, so no update is lost. A reconcile timeout re-reads the range +even without a watch fire, bounding staleness for writes from pre-upgrade +components that don't bump counters. + +This module owns the *mechanism* only. It emits typed :class:`ChangeEvent` +batches (``created``/``updated``/``deleted`` over model objects); building wire +DTOs and framing them as SSE is the web layer's concern (``_sse.py``). Keeping +the change-detection here — behind the ``ChangeEvent`` interface — means the +counter-and-re-read backend can later be swapped for a version-index or +change-log backend without touching controllers or the API. +""" + +import asyncio +import json +import logging +import threading +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type + +import fdb + +from simplyblock_core import watches + +logger = logging.getLogger(__name__) + +# Upper bound on staleness for entity writes that don't bump the change counter +# (processes from before the counter was introduced). +WATCH_RECONCILE_SEC = 30.0 + +_FDB_RETRY_BACKOFF_SEC = (1.0, 2.0, 4.0) + +_UNSET: Any = object() + +# Filters the shared full model list down to one subscription's scope. Returns +# model objects (typically by reusing a DBController getter with ``source=``). +Select = Callable[[List[Any]], List[Any]] + + +class WatchUnavailable(Exception): + """The watch backend failed irrecoverably; the stream must terminate.""" + + +@dataclass +class ChangeEvent: + """A single entity transition within a watched, scoped set.""" + + kind: str # 'created' | 'updated' | 'deleted' + id: str + # For 'deleted': the entity's current (unfiltered) model if it left the + # scoped set but is still retrievable (e.g. an LVol whose status flipped to + # 'deleted'), else None once it is physically gone. + model: Optional[Any] + + +def diff_by_id( + prev: Dict[str, dict], new: Dict[str, Any], full: Dict[str, Any], +) -> Tuple[List[ChangeEvent], Dict[str, dict]]: + """Diff the scoped set against the previous emission, keyed by ``get_id()``. + + ``prev`` maps id -> the ``to_dict()`` snapshot emitted last time; ``new`` and + ``full`` map id -> model for the current scoped and unscoped sets. Returns + the events (created/updated in ``new`` iteration order, deletions appended) + and the fresh id -> ``to_dict()`` snapshot to carry forward. + """ + events: List[ChangeEvent] = [] + new_dicts: Dict[str, dict] = {} + for entity_id, model in new.items(): + snapshot = model.to_dict() + new_dicts[entity_id] = snapshot + if entity_id not in prev: + events.append(ChangeEvent('created', entity_id, model)) + elif snapshot != prev[entity_id]: + events.append(ChangeEvent('updated', entity_id, model)) + # else unchanged: no event + for entity_id in prev: + if entity_id in new: + continue + # Left the scoped set. If still present in the unscoped set, carry its + # current model so the wire layer can render the final representation; + # otherwise it is physically gone (model=None -> empty representation). + events.append(ChangeEvent('deleted', entity_id, full.get(entity_id))) + return events, new_dicts + + +class _Subscription: + """Coalescing mailbox: written by the hub thread, read by one stream.""" + + def __init__(self, loop: asyncio.AbstractEventLoop, notify: asyncio.Event) -> None: + self._loop = loop + self._notify = notify + self._latest: Any = _UNSET + self._dirty = False + self.failed = False + + def _deliver(self, state: Any) -> None: # event-loop thread + self._latest = state + self._dirty = True + self._notify.set() + + def _fail(self) -> None: # event-loop thread + self.failed = True + self._notify.set() + + def deliver(self, state: Any) -> None: # hub thread + self._loop.call_soon_threadsafe(self._deliver, state) + + def fail(self) -> None: # hub thread + self._loop.call_soon_threadsafe(self._fail) + + def take(self) -> Any: + """Latest state if there is an unconsumed publication, else ``_UNSET``.""" + if not self._dirty: + return _UNSET + self._dirty = False + return self._latest + + +def _watch_tx(tr: Any, key: bytes) -> Any: + return tr.watch(key) + + +class WatchHub: + """Single FDB watch + shared range re-read for one watched model class. + + Delivers ``List[model]`` (the full current class range) to subscribers. + Change detection compares the *raw* parsed records (cheap) so the reconcile + re-read only republishes on a real change; models are constructed only when + something changed and shared across all subscribers. + """ + + def __init__(self, model_cls: Type[Any]) -> None: + instance = model_cls() + self._model_cls = model_cls + self._name = model_cls.__name__ + # Built from parts rather than get_db_id(): a fresh JobSchedule's + # compound get_id() is not empty, which would corrupt the prefix. + self._prefix = f'{instance.object_type}/{instance.name}/'.encode() + self._counter_key = watches.watch_counter_key(model_cls) + self._lock = threading.Lock() + self._subs: set = set() + self._cache: Any = _UNSET # last published List[model] (for late joiners) + self._raw: Any = _UNSET # last raw dict[key, dict] (for change detection) + self._stop = threading.Event() + self._wake = threading.Event() + self._thread: Optional[threading.Thread] = None + + def subscribe(self, loop: asyncio.AbstractEventLoop, notify: asyncio.Event) -> _Subscription: + sub = _Subscription(loop, notify) + with self._lock: + alive = self._thread is not None and self._thread.is_alive() and not self._stop.is_set() + if not alive: + self._cache = _UNSET + self._raw = _UNSET + self._stop = threading.Event() + self._wake = threading.Event() + self._thread = threading.Thread( + target=self._run, args=(self._stop, self._wake), + daemon=True, name=f'watch-{self._name}', + ) + self._thread.start() + self._subs.add(sub) + if self._cache is not _UNSET: + # subscribe() runs on the event loop thread, so deliver directly. + sub._deliver(self._cache) + return sub + + def unsubscribe(self, sub: _Subscription) -> None: + with self._lock: + self._subs.discard(sub) + if not self._subs: + self._stop.set() + self._wake.set() + + def subscriber_count(self) -> int: + with self._lock: + return len(self._subs) + + def _publish(self, raw: Dict[Any, dict], models: List[Any]) -> None: + with self._lock: + self._raw = raw + self._cache = models + subs = list(self._subs) + for sub in subs: + sub.deliver(models) + + def _fail_all(self) -> None: + with self._lock: + subs = list(self._subs) + for sub in subs: + sub.fail() + + def _read_raw(self, kv_store: Any) -> Dict[Any, dict]: + state: Dict[Any, dict] = {} + for k, v in kv_store.get_range_startswith(self._prefix): + try: + state[bytes(k)] = json.loads(v) + except ValueError: + logger.warning('Skipping unparsable record %s', bytes(k)) + return state + + def _to_models(self, raw: Dict[Any, dict]) -> List[Any]: + return [self._model_cls().from_dict(d) for d in raw.values()] + + def _run(self, stop: threading.Event, wake: threading.Event) -> None: + from simplyblock_core.db_controller import DBController + kv_store = DBController().kv_store + set_watch = fdb.transactional(_watch_tx) + failures = 0 + while not stop.is_set(): + watch = None + try: + # Watch first, then read: see module docstring. + watch = set_watch(kv_store, self._counter_key) + raw = self._read_raw(kv_store) + failures = 0 + with self._lock: + changed = self._raw is _UNSET or raw != self._raw + if changed: + self._publish(raw, self._to_models(raw)) + wake.clear() + # Runs on the FDB network thread: must only set the event, + # never block or call back into fdb. + watch.on_ready(lambda _f: wake.set()) + wake.wait(timeout=WATCH_RECONCILE_SEC) + except Exception: + logger.exception('Watch hub for %s failed', self._name) + failures += 1 + if failures > len(_FDB_RETRY_BACKOFF_SEC): + self._fail_all() + return + stop.wait(timeout=_FDB_RETRY_BACKOFF_SEC[failures - 1]) + finally: + if watch is not None: + try: + watch.cancel() + except Exception: + # Best-effort cleanup: cancellation failures should not + # mask the main loop error/retry behavior. + logger.debug('Ignoring watch cancellation failure for %s', self._name, exc_info=True) + + +_hubs: Dict[type, WatchHub] = {} +_hubs_lock = threading.Lock() + + +def get_hub(model_cls: Type[Any]) -> WatchHub: + with _hubs_lock: + hub = _hubs.get(model_cls) + if hub is None: + hub = _hubs[model_cls] = WatchHub(model_cls) + return hub + + +def backend_available() -> bool: + """Whether the watch backend can serve streams (the DB is reachable).""" + from simplyblock_core.db_controller import DBController + return DBController().kv_store is not None + + +async def watch( + model_cls: Type[Any], + *, + select: Optional[Select] = None, + ancestors: Sequence[Tuple[Type[Any], str]] = (), +): + """Async stream of :class:`ChangeEvent` batches for one watched class. + + ``select`` filters the shared full model list to this subscription's scope + (reuse a DBController getter with ``source=`` so the filter is defined once); + the diff runs on its output, so entities entering/leaving the scoped set emit + ``created``/``deleted``. ``ancestors`` are ``(model_cls, id)`` pairs whose + disappearance closes the stream (a reconnect then hits the endpoint's regular + 404). The first yielded batch is the initial state (all ``created``). + + Raises :class:`WatchUnavailable` if the backend fails irrecoverably. + """ + loop = asyncio.get_running_loop() + notify = asyncio.Event() + main_sub = get_hub(model_cls).subscribe(loop, notify) + ancestor_subs = [ + (get_hub(cls).subscribe(loop, notify), cls, str(ancestor_id)) + for cls, ancestor_id in ancestors + ] + try: + prev: Dict[str, dict] = {} + while True: + await notify.wait() + notify.clear() + + if main_sub.failed or any(sub.failed for sub, _, _ in ancestor_subs): + raise WatchUnavailable() + + for ancestor_sub, _, ancestor_id in ancestor_subs: + ancestor_state = ancestor_sub.take() + if ancestor_state is _UNSET: + continue + if not any(m.get_id() == ancestor_id for m in ancestor_state): + return + + models = main_sub.take() + if models is _UNSET: + continue + # select() filtering and to_dict() diffing are CPU-bound; keep them + # off the event loop. Selects must not do blocking DB reads (they + # filter the supplied model list) — DTO building, which may, stays + # in the web layer's threadpool. + events, prev = await loop.run_in_executor( + None, _compute_batch, models, select, prev) + yield events + finally: + get_hub(model_cls).unsubscribe(main_sub) + for ancestor_sub, cls, _ in ancestor_subs: + get_hub(cls).unsubscribe(ancestor_sub) + + +def _compute_batch( + models: List[Any], select: Optional[Select], prev: Dict[str, dict], +) -> Tuple[List[ChangeEvent], Dict[str, dict]]: + full = {m.get_id(): m for m in models} + selected = select(models) if select is not None else models + new = {m.get_id(): m for m in selected} + return diff_by_id(prev, new, full) diff --git a/simplyblock_web/api/v2/_sse.py b/simplyblock_web/api/v2/_sse.py new file mode 100644 index 000000000..7a504a7e8 --- /dev/null +++ b/simplyblock_web/api/v2/_sse.py @@ -0,0 +1,162 @@ +# coding=utf-8 +"""SSE framing for the v2 API watch streams (``?watch=true``). + +Consumes a controller's :class:`~simplyblock_core.watch.ChangeEvent` stream and +frames it as Server-Sent Events: a ``snapshot`` event with the current state +first, then ``created``/``updated``/``deleted`` events carrying the full +resource representation. A ``deleted`` event carries the resource's final state +when it is still retrievable, or an empty object once it is gone entirely. + +This layer owns only the wire concerns — building DTOs, suppressing changes that +are invisible in the DTO, stream lifetime, ping/retry, and the SSE envelope. It +knows nothing about FoundationDB, watches, filtering, or raw records; those live +in :mod:`simplyblock_core.watch` and the controllers. +""" + +import asyncio +import logging +import time +from typing import Any, Callable, Dict, List, Tuple + +from fastapi import HTTPException, Query +from sse_starlette import EventSourceResponse, ServerSentEvent +from starlette.concurrency import run_in_threadpool +from typing_extensions import Annotated + +from simplyblock_core.watch import ChangeEvent, WatchUnavailable, backend_available + +logger = logging.getLogger(__name__) + +# Builds the wire DTO from a model. May do blocking DB reads — only ever called +# via run_in_threadpool. +DTOBuilder = Callable[[Any], Any] + +# Streams are closed after this long; bearer tokens are only validated at request +# start, so this bounds how long a revoked token keeps a live stream (clients +# re-authenticate on reconnect, like Kubernetes watch timeouts). +WATCH_MAX_LIFETIME_SEC = 3600.0 + +SSE_RETRY_MS = 3000 +PING_SEC = 15 + +# OpenAPI ``responses`` snippet for routes that also serve ``?watch=true``. +WATCH_RESPONSES: Dict[Any, Any] = { + 200: {'content': {'text/event-stream': {'schema': {'type': 'string'}}}}, +} + +# Query-parameter documentation, shared by all watchable routes. +WATCH_PARAM_DESCRIPTION = ( + 'Stream state changes as Server-Sent Events instead of returning a plain ' + 'response: a `snapshot` event with the current state first, then ' + '`created`/`updated`/`deleted` events carrying the full resource ' + 'representation. A `deleted` event carries the resource\'s final state ' + 'when it is still retrievable (e.g. a volume whose status became ' + '`deleted`), or an empty object once it is gone entirely. Streams do not ' + 'support resume; reconnecting clients receive a fresh snapshot. Changes ' + 'written by pre-upgrade components may take up to 30 seconds to appear.' +) + +# ``watch: WatchParam = False`` on a route adds the documented query flag. +WatchParam = Annotated[bool, Query(description=WATCH_PARAM_DESCRIPTION)] + + +def _build_snapshot( + events: List[ChangeEvent], build_dto: DTOBuilder, single: bool, +) -> Tuple[List[ServerSentEvent], Dict[str, str]]: + """Render the initial batch (all ``created``) as one ``snapshot`` event.""" + cache = {event.id: build_dto(event.model).model_dump_json() for event in events} + if single: + if not cache: + return [], cache + data = next(iter(cache.values())) + else: + data = '[' + ','.join(cache[event.id] for event in events) + ']' + return [ServerSentEvent(event='snapshot', data=data, retry=SSE_RETRY_MS)], cache + + +def _build_events( + events: List[ChangeEvent], cache: Dict[str, str], build_dto: DTOBuilder, single: bool, +) -> Tuple[List[ServerSentEvent], Dict[str, str], bool]: + """Render a diff batch as individual ``created``/``updated``/``deleted`` events.""" + out: List[ServerSentEvent] = [] + new_cache = dict(cache) + deleted = False + for event in events: + if event.kind == 'deleted': + deleted = True + if event.model is not None: + try: + data = build_dto(event.model).model_dump_json() + except Exception: + logger.exception('Failed to build DTO for deleted entity %s', event.id) + data = new_cache.get(event.id, '{}') + else: + data = '{}' + new_cache.pop(event.id, None) + out.append(ServerSentEvent(event='deleted', data=data)) + else: + data = build_dto(event.model).model_dump_json() + if event.kind == 'updated' and data == new_cache.get(event.id): + # Change invisible in the DTO (e.g. a JobSchedule lease + # heartbeat rewrite): suppress it. + continue + new_cache[event.id] = data + out.append(ServerSentEvent(event='updated' if single else event.kind, data=data)) + return out, new_cache, deleted + + +def sse_response(change_stream, build_dto: DTOBuilder, *, single: bool = False) -> EventSourceResponse: + """Frame a controller's ChangeEvent stream as an SSE response. + + ``change_stream`` is an async iterator of ``List[ChangeEvent]`` batches (the + first batch is the initial state). ``build_dto`` converts a model into its + wire DTO. ``single`` switches to detail semantics: a single-object snapshot + and closing the stream once the entity is deleted. + """ + if not backend_available(): + raise HTTPException(503, 'Database unavailable') + + async def event_stream() -> Any: + dto_cache: Dict[str, str] = {} + first = True + deadline = time.monotonic() + WATCH_MAX_LIFETIME_SEC + try: + while True: + timeout = deadline - time.monotonic() + if timeout <= 0: + return + try: + events = await asyncio.wait_for(change_stream.__anext__(), timeout) + except asyncio.TimeoutError: + return + except StopAsyncIteration: + return + except WatchUnavailable: + yield ServerSentEvent(event='error', data='{"detail": "backend unavailable"}') + return + + if first: + first = False + sse_events, dto_cache = await run_in_threadpool( + _build_snapshot, events, build_dto, single) + if single and not dto_cache: + # Deleted between dependency resolution and first + # publication; close so the reconnect 404s. + return + for event in sse_events: + yield event + else: + sse_events, dto_cache, deleted = await run_in_threadpool( + _build_events, events, dto_cache, build_dto, single) + for event in sse_events: + yield event + if single and deleted: + return + finally: + await change_stream.aclose() + + return EventSourceResponse( + event_stream(), + ping=PING_SEC, + headers={'Cache-Control': 'no-store', 'X-Accel-Buffering': 'no'}, + ) diff --git a/tests/unit/test_watch_core.py b/tests/unit/test_watch_core.py new file mode 100644 index 000000000..963aba7e3 --- /dev/null +++ b/tests/unit/test_watch_core.py @@ -0,0 +1,197 @@ +# coding=utf-8 +"""Core watch primitive: diff_by_id and the async watch() generator.""" + +import asyncio + +import pytest + +from simplyblock_core import watch as watchmod +from simplyblock_core.watch import ( + WatchUnavailable, _Subscription, diff_by_id, watch, +) + + +class FakeModel: + def __init__(self, entity_id, **fields): + self._id = entity_id + self._fields = fields + + def get_id(self): + return self._id + + def to_dict(self): + return {'uuid': self._id, **self._fields} + + +def _m(entity_id, **fields): + return FakeModel(entity_id, **fields) + + +# ---- diff_by_id (pure function) ---- + +def test_diff_created_updated_deleted(): + prev = {'a': _m('a', status='x').to_dict(), 'b': _m('b', status='x').to_dict()} + b2, c = _m('b', status='y'), _m('c', status='x') + new = {'b': b2, 'c': c} + events, new_dicts = diff_by_id(prev, new, dict(new)) + by_kind = {e.kind: e for e in events} + assert set(by_kind) == {'created', 'updated', 'deleted'} + assert by_kind['created'].id == 'c' + assert by_kind['updated'].id == 'b' and by_kind['updated'].model is b2 + assert by_kind['deleted'].id == 'a' + assert by_kind['deleted'].model is None # 'a' absent from full: physically gone + assert set(new_dicts) == {'b', 'c'} + + +def test_diff_unchanged_emits_nothing(): + prev = {'a': _m('a', status='x').to_dict()} + events, _ = diff_by_id(prev, {'a': _m('a', status='x')}, {'a': _m('a', status='x')}) + assert events == [] + + +def test_diff_scope_leave_carries_current_state(): + """Left the scoped set but still present in full -> deleted carries the + current (unscoped) model, not a stale one.""" + prev = {'a': _m('a', status='online').to_dict()} + current = _m('a', status='deleted') + events, _ = diff_by_id(prev, {}, {'a': current}) + [event] = events + assert event.kind == 'deleted' and event.model is current + + +def test_diff_created_order_follows_new(): + events, _ = diff_by_id({}, {'a': _m('a'), 'b': _m('b')}, {'a': _m('a'), 'b': _m('b')}) + assert [e.id for e in events] == ['a', 'b'] + + +# ---- watch() generator with a fake hub ---- + +class FakeHub: + def __init__(self): + self.subs = [] + self.cache = None + + def subscribe(self, loop, notify): + sub = _Subscription(loop, notify) + self.subs.append(sub) + if self.cache is not None: + sub._deliver(self.cache) + return sub + + def unsubscribe(self, sub): + self.subs.remove(sub) + + def publish(self, models): + self.cache = models + for sub in list(self.subs): + sub._deliver(models) + + def fail(self): + for sub in list(self.subs): + sub._fail() + + +class Thing: + pass + + +class Parent: + pass + + +def _patch(monkeypatch, hubs): + monkeypatch.setattr(watchmod, 'get_hub', lambda cls: hubs[cls]) + + +async def _next(gen, timeout=2.0): + return await asyncio.wait_for(gen.__anext__(), timeout) + + +def test_watch_snapshot_update_delete(monkeypatch): + hub = FakeHub() + _patch(monkeypatch, {Thing: hub}) + hub.publish([_m('a', status='online')]) + + async def scenario(): + gen = watch(Thing) + [event] = await _next(gen) + assert event.kind == 'created' and event.id == 'a' + + hub.publish([_m('a', status='offline')]) + [event] = await _next(gen) + assert event.kind == 'updated' and event.model.to_dict()['status'] == 'offline' + + hub.publish([]) + [event] = await _next(gen) + assert event.kind == 'deleted' and event.id == 'a' + + await gen.aclose() + assert hub.subs == [] + + asyncio.run(scenario()) + + +def test_watch_coalesces_bursts(monkeypatch): + hub = FakeHub() + _patch(monkeypatch, {Thing: hub}) + hub.publish([_m('a', status='s0')]) + + async def scenario(): + gen = watch(Thing) + assert (await _next(gen))[0].kind == 'created' + + # Two publications before the generator runs: only the latest diffs. + hub.publish([_m('a', status='s1')]) + hub.publish([_m('a', status='s2')]) + [event] = await _next(gen) + assert event.kind == 'updated' and event.model.to_dict()['status'] == 's2' + + with pytest.raises(asyncio.TimeoutError): + await _next(gen, timeout=0.2) + + asyncio.run(scenario()) + + +def test_watch_scope_select(monkeypatch): + hub = FakeHub() + _patch(monkeypatch, {Thing: hub}) + hub.publish([_m('a', keep=True), _m('b', keep=False)]) + + async def scenario(): + gen = watch(Thing, select=lambda models: [m for m in models if m.to_dict()['keep']]) + events = await _next(gen) + assert [e.id for e in events] == ['a'] + + asyncio.run(scenario()) + + +def test_watch_ancestor_deletion_closes(monkeypatch): + hub, parent = FakeHub(), FakeHub() + _patch(monkeypatch, {Thing: hub, Parent: parent}) + hub.publish([_m('a')]) + parent.publish([_m('p')]) + + async def scenario(): + gen = watch(Thing, ancestors=[(Parent, 'p')]) + assert (await _next(gen))[0].id == 'a' + parent.publish([]) + with pytest.raises(StopAsyncIteration): + await _next(gen) + assert hub.subs == [] and parent.subs == [] + + asyncio.run(scenario()) + + +def test_watch_failure_raises(monkeypatch): + hub = FakeHub() + _patch(monkeypatch, {Thing: hub}) + hub.publish([_m('a')]) + + async def scenario(): + gen = watch(Thing) + await _next(gen) + hub.fail() + with pytest.raises(WatchUnavailable): + await _next(gen) + + asyncio.run(scenario()) diff --git a/tests/unit/web/api/v2/conftest.py b/tests/unit/web/api/v2/conftest.py index def323de8..782c9e3d2 100644 --- a/tests/unit/web/api/v2/conftest.py +++ b/tests/unit/web/api/v2/conftest.py @@ -109,6 +109,55 @@ def client(app): return TestClient(app) +# --- Watch (?watch=true) dispatch ------------------------------------------- + +class _FakeChangeStream: + """Async-iterator stub over pre-built ``ChangeEvent`` batches. + + Yields each batch once, then raises ``StopAsyncIteration`` so the SSE + generator returns and the streaming response completes for ``TestClient``. + """ + + def __init__(self, batches): + self._it = iter(batches) + self.closed = False + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._it) + except StopIteration: + raise StopAsyncIteration + + async def aclose(self): + self.closed = True + + +@pytest.fixture() +def watch_stream(monkeypatch): + """Enable the SSE backend and build fake ``ChangeEvent`` streams. + + Returns a factory: ``watch_stream([model, ...])`` yields a single + ``created`` batch for the given models (rendered as one ``snapshot`` event) + and then closes. This lets the endpoint tests assert the ``?watch=true`` + dispatch — the correct controller ``watch_*`` call and a real + ``text/event-stream`` snapshot built by the route's own DTO builder — + without a live watch backend. + """ + from simplyblock_core.watch import ChangeEvent + from simplyblock_web.api.v2 import _sse + + monkeypatch.setattr(_sse, 'backend_available', lambda: True) + + def make(models): + batch = [ChangeEvent('created', model.get_id(), model) for model in models] + return _FakeChangeStream([batch]) + + return make + + # --- Controller mocks ------------------------------------------------------- @pytest.fixture() @@ -160,6 +209,7 @@ def storage_node_ops(monkeypatch): def tasks_controller(monkeypatch): mock = MagicMock() monkeypatch.setattr(storage_node_module, 'tasks_controller', mock) + monkeypatch.setattr(task_module, 'tasks_controller', mock) return mock diff --git a/tests/unit/web/api/v2/test_cluster_endpoints.py b/tests/unit/web/api/v2/test_cluster_endpoints.py index d9ff1d2f7..0abbba54f 100644 --- a/tests/unit/web/api/v2/test_cluster_endpoints.py +++ b/tests/unit/web/api/v2/test_cluster_endpoints.py @@ -202,3 +202,28 @@ def test_calls_add_replication(self, client, cluster, cluster_ops): timeout=30, target_pool='pool-1', ) + + +class TestWatchClusters: + + def test_list_dispatches_watch_clusters(self, client, cluster, cluster_ops, watch_stream): + cluster_ops.watch_clusters.return_value = watch_stream([cluster]) + + response = client.get('/api/v2/clusters/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert CLUSTER_ID in response.text + cluster_ops.watch_clusters.assert_called_once_with() + + def test_detail_dispatches_watch_cluster(self, client, cluster, cluster_ops, watch_stream): + cluster_ops.watch_cluster.return_value = watch_stream([cluster]) + + response = client.get(f'/api/v2/clusters/{CLUSTER_ID}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert CLUSTER_ID in response.text + cluster_ops.watch_cluster.assert_called_once_with(CLUSTER_ID) diff --git a/tests/unit/web/api/v2/test_device_endpoints.py b/tests/unit/web/api/v2/test_device_endpoints.py index 88d718999..311c603b1 100644 --- a/tests/unit/web/api/v2/test_device_endpoints.py +++ b/tests/unit/web/api/v2/test_device_endpoints.py @@ -78,3 +78,29 @@ def test_iostats(self, client, device, device_controller): assert response.status_code == 200 device_controller.get_device_iostats.assert_called_once_with( DEVICE_ID, None, parse_sizes=False) + + +class TestWatchDevices: + + def test_list_dispatches_watch_devices(self, client, device, device_controller, watch_stream): + device_controller.watch_devices.return_value = watch_stream([device]) + + response = client.get(f'{BASE}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert DEVICE_ID in response.text + device_controller.watch_devices.assert_called_once_with(CLUSTER_ID, STORAGE_NODE_ID) + + def test_detail_dispatches_watch_device(self, client, device, device_controller, watch_stream): + device_controller.watch_device.return_value = watch_stream([device]) + + response = client.get(f'{BASE}/{DEVICE_ID}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert DEVICE_ID in response.text + device_controller.watch_device.assert_called_once_with( + CLUSTER_ID, STORAGE_NODE_ID, DEVICE_ID) diff --git a/tests/unit/web/api/v2/test_snapshot_endpoints.py b/tests/unit/web/api/v2/test_snapshot_endpoints.py index 0336ff92b..8ef79ec84 100644 --- a/tests/unit/web/api/v2/test_snapshot_endpoints.py +++ b/tests/unit/web/api/v2/test_snapshot_endpoints.py @@ -53,3 +53,29 @@ def test_deletes_snapshot(self, client, snapshot, snapshot_controller): assert response.status_code == 204 snapshot_controller.delete.assert_called_once_with(SNAPSHOT_ID) + + +class TestWatchSnapshots: + + def test_list_dispatches_watch_snapshots(self, client, snapshot, snapshot_controller, watch_stream): + snapshot_controller.watch_snapshots.return_value = watch_stream([snapshot]) + + response = client.get(f'{BASE}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert SNAPSHOT_ID in response.text + snapshot_controller.watch_snapshots.assert_called_once_with(CLUSTER_ID, POOL_ID) + + def test_detail_dispatches_watch_snapshot(self, client, snapshot, snapshot_controller, watch_stream): + snapshot_controller.watch_snapshot.return_value = watch_stream([snapshot]) + + response = client.get(f'{BASE}/{SNAPSHOT_ID}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert SNAPSHOT_ID in response.text + snapshot_controller.watch_snapshot.assert_called_once_with( + CLUSTER_ID, POOL_ID, SNAPSHOT_ID) diff --git a/tests/unit/web/api/v2/test_storage_node_endpoints.py b/tests/unit/web/api/v2/test_storage_node_endpoints.py index e543a8a70..c85a9e653 100644 --- a/tests/unit/web/api/v2/test_storage_node_endpoints.py +++ b/tests/unit/web/api/v2/test_storage_node_endpoints.py @@ -148,3 +148,28 @@ def test_capacity_passes_history(self, client, storage_node, storage_node_ops): assert response.status_code == 200 storage_node_ops.get_node_iostats_history.assert_called_once_with( STORAGE_NODE_ID, '10', parse_sizes=False, with_sizes=True) + + +class TestWatchStorageNodes: + + def test_list_dispatches_watch_storage_nodes(self, client, storage_node, storage_node_ops, watch_stream): + storage_node_ops.watch_storage_nodes.return_value = watch_stream([storage_node]) + + response = client.get(f'{BASE}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert STORAGE_NODE_ID in response.text + storage_node_ops.watch_storage_nodes.assert_called_once_with(CLUSTER_ID) + + def test_detail_dispatches_watch_storage_node(self, client, storage_node, storage_node_ops, watch_stream): + storage_node_ops.watch_storage_node.return_value = watch_stream([storage_node]) + + response = client.get(f'{BASE}/{STORAGE_NODE_ID}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert STORAGE_NODE_ID in response.text + storage_node_ops.watch_storage_node.assert_called_once_with(CLUSTER_ID, STORAGE_NODE_ID) diff --git a/tests/unit/web/api/v2/test_storage_pool_endpoints.py b/tests/unit/web/api/v2/test_storage_pool_endpoints.py index e0903c4e6..8331eb0da 100644 --- a/tests/unit/web/api/v2/test_storage_pool_endpoints.py +++ b/tests/unit/web/api/v2/test_storage_pool_endpoints.py @@ -121,3 +121,28 @@ def test_remove_host(self, client, pool, pool_controller): assert response.status_code == 204 pool_controller.remove_host_from_pool.assert_called_once_with(POOL_ID, HOST_NQN) + + +class TestWatchStoragePools: + + def test_list_dispatches_watch_pools(self, client, pool, pool_controller, watch_stream): + pool_controller.watch_pools.return_value = watch_stream([pool]) + + response = client.get(f'{BASE}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert POOL_ID in response.text + pool_controller.watch_pools.assert_called_once_with(CLUSTER_ID) + + def test_detail_dispatches_watch_pool(self, client, pool, pool_controller, watch_stream): + pool_controller.watch_pool.return_value = watch_stream([pool]) + + response = client.get(f'{BASE}/{POOL_ID}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert POOL_ID in response.text + pool_controller.watch_pool.assert_called_once_with(CLUSTER_ID, POOL_ID) diff --git a/tests/unit/web/api/v2/test_task_endpoints.py b/tests/unit/web/api/v2/test_task_endpoints.py index a7e61e483..73c88420e 100644 --- a/tests/unit/web/api/v2/test_task_endpoints.py +++ b/tests/unit/web/api/v2/test_task_endpoints.py @@ -55,3 +55,28 @@ def test_unknown_task_returns_404(self, client, db, cluster): response = client.get(f'{BASE}/{TASK_ID}/') assert response.status_code == 404 + + +class TestWatchTasks: + + def test_list_dispatches_watch_tasks(self, client, task, tasks_controller, watch_stream): + tasks_controller.watch_tasks.return_value = watch_stream([task]) + + response = client.get(f'{BASE}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert TASK_ID in response.text + tasks_controller.watch_tasks.assert_called_once_with(CLUSTER_ID) + + def test_detail_dispatches_watch_task(self, client, task, tasks_controller, watch_stream): + tasks_controller.watch_task.return_value = watch_stream([task]) + + response = client.get(f'{BASE}/{TASK_ID}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert TASK_ID in response.text + tasks_controller.watch_task.assert_called_once_with(CLUSTER_ID, TASK_ID) diff --git a/tests/unit/web/api/v2/test_volume_endpoints.py b/tests/unit/web/api/v2/test_volume_endpoints.py index ca12385f3..52c753f1d 100644 --- a/tests/unit/web/api/v2/test_volume_endpoints.py +++ b/tests/unit/web/api/v2/test_volume_endpoints.py @@ -232,3 +232,28 @@ def test_remove_host(self, client, volume, lvol_controller): assert response.status_code == 204 lvol_controller.remove_host_from_lvol.assert_called_once_with( VOLUME_ID, 'nqn.2014-08.org.nvmexpress:host-1') + + +class TestWatchVolumes: + + def test_list_dispatches_watch_volumes(self, client, volume, lvol_controller, watch_stream): + lvol_controller.watch_volumes.return_value = watch_stream([volume]) + + response = client.get(f'{BASE}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert VOLUME_ID in response.text + lvol_controller.watch_volumes.assert_called_once_with(CLUSTER_ID, POOL_ID) + + def test_detail_dispatches_watch_volume(self, client, volume, lvol_controller, watch_stream): + lvol_controller.watch_volume.return_value = watch_stream([volume]) + + response = client.get(f'{BASE}/{VOLUME_ID}/?watch=true') + + assert response.status_code == 200 + assert response.headers['content-type'].startswith('text/event-stream') + assert 'event: snapshot' in response.text + assert VOLUME_ID in response.text + lvol_controller.watch_volume.assert_called_once_with(CLUSTER_ID, POOL_ID, VOLUME_ID)