From 35e0aa45512c2d8b809fb1397e90990b61fb822a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:21:03 +0000 Subject: [PATCH 1/4] =?UTF-8?q?Phase=201:=20Research=20Prototype=20?= =?UTF-8?q?=E2=80=94=20complete=20implementation=20with=2064=20passing=20t?= =?UTF-8?q?ests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 30 ++ .gitignore | 18 + README.md | 239 ++++++++++- main.py | 160 ++++++++ requirements.txt | 63 +++ src/__init__.py | 9 + src/agents/__init__.py | 3 + src/agents/compliance_agent.py | 142 +++++++ src/agents/decision_parliament.py | 176 ++++++++ src/agents/institutional_footprint.py | 112 +++++ src/agents/market_dna_detector.py | 133 ++++++ src/agents/models.py | 92 +++++ src/agents/order_book_analyst.py | 92 +++++ src/agents/risk_governor.py | 124 ++++++ src/audit/__init__.py | 3 + src/audit/ledger.py | 114 ++++++ src/chatbot/__init__.py | 3 + src/chatbot/interface.py | 359 +++++++++++++++++ src/config.py | 67 +++ src/dashboard/__init__.py | 3 + src/dashboard/app.py | 304 ++++++++++++++ src/data_intake/__init__.py | 3 + src/data_intake/data_integrity_agent.py | 127 ++++++ src/data_intake/models.py | 131 ++++++ src/data_intake/polygon_feed.py | 132 ++++++ src/data_intake/sample_feed.py | 154 +++++++ src/features/__init__.py | 3 + src/features/engineer.py | 444 ++++++++++++++++++++ src/orchestrator.py | 166 ++++++++ src/order_book/__init__.py | 3 + src/order_book/engine.py | 516 ++++++++++++++++++++++++ src/utils/__init__.py | 3 + tests/__init__.py | 3 + tests/test_agents.py | 255 ++++++++++++ tests/test_data_models.py | 79 ++++ tests/test_features.py | 82 ++++ tests/test_orchestrator.py | 91 +++++ tests/test_order_book_engine.py | 97 +++++ 38 files changed, 4533 insertions(+), 2 deletions(-) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 src/__init__.py create mode 100644 src/agents/__init__.py create mode 100644 src/agents/compliance_agent.py create mode 100644 src/agents/decision_parliament.py create mode 100644 src/agents/institutional_footprint.py create mode 100644 src/agents/market_dna_detector.py create mode 100644 src/agents/models.py create mode 100644 src/agents/order_book_analyst.py create mode 100644 src/agents/risk_governor.py create mode 100644 src/audit/__init__.py create mode 100644 src/audit/ledger.py create mode 100644 src/chatbot/__init__.py create mode 100644 src/chatbot/interface.py create mode 100644 src/config.py create mode 100644 src/dashboard/__init__.py create mode 100644 src/dashboard/app.py create mode 100644 src/data_intake/__init__.py create mode 100644 src/data_intake/data_integrity_agent.py create mode 100644 src/data_intake/models.py create mode 100644 src/data_intake/polygon_feed.py create mode 100644 src/data_intake/sample_feed.py create mode 100644 src/features/__init__.py create mode 100644 src/features/engineer.py create mode 100644 src/orchestrator.py create mode 100644 src/order_book/__init__.py create mode 100644 src/order_book/engine.py create mode 100644 src/utils/__init__.py create mode 100644 tests/__init__.py create mode 100644 tests/test_agents.py create mode 100644 tests/test_data_models.py create mode 100644 tests/test_features.py create mode 100644 tests/test_orchestrator.py create mode 100644 tests/test_order_book_engine.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..bf00a20 --- /dev/null +++ b/.env.example @@ -0,0 +1,30 @@ +# Environment configuration for Institutional Microstructure Intelligence System +# Copy this file to .env and fill in your credentials. +# NEVER commit .env to version control. + +# ── Data Feed Credentials ────────────────────────────────────────────────────── +POLYGON_API_KEY=your_polygon_api_key_here +ALPACA_API_KEY=your_alpaca_api_key_here +ALPACA_SECRET_KEY=your_alpaca_secret_key_here +ALPACA_BASE_URL=https://paper-api.alpaca.markets # paper by default + +# ── LLM API Keys (optional – leave blank to use local LLM) ──────────────────── +OPENAI_API_KEY= +ANTHROPIC_API_KEY= + +# ── Storage ─────────────────────────────────────────────────────────────────── +DATABASE_URL=******localhost:5432/microstructure +REDIS_URL=redis://localhost:6379/0 + +# ── System Defaults ─────────────────────────────────────────────────────────── +EXECUTION_MODE=paper # paper | research | live (live requires explicit override) +AUDIT_LOG_DIR=data/audit_logs +DEFAULT_SYMBOL=AAPL +DEFAULT_FEED=polygon # polygon | alpaca | ibkr | sample + +# ── Risk Defaults ───────────────────────────────────────────────────────────── +MAX_DAILY_LOSS_PCT=2.0 +MAX_POSITION_SIZE_PCT=5.0 +MAX_SPREAD_MULTIPLE=3.0 # reject if spread > N × 20-day average spread +MIN_MODEL_CONFIDENCE=0.55 +MIN_DATA_QUALITY_SCORE=0.80 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..547a896 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +.env +*.pyc +__pycache__/ +*.egg-info/ +dist/ +build/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +*.log +data/audit_logs/ +data/sample/*.parquet +*.pkl +*.joblib +*.h5 +*.pt +.DS_Store +Thumbs.db diff --git a/README.md b/README.md index b5826e1..f9bf5fd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,237 @@ -# Institutional-Microstructure- -My liberty of Code to my Scripts 🗽 +# Institutional Microstructure Intelligence System +### Phase 1 — Research Prototype | Paper-Only | Compliance-Aware | Audit-Driven + +> **"Evidence first. Risk second. Execution last."** + +A sophisticated, institutional-grade AI Agentic Trading Intelligence System for Level 2 order book analysis, market microstructure intelligence, behavioural order-flow inference, and risk-controlled decision support. + +--- + +## What This Is + +This system analyses Level 2 order book data, market depth, time-and-sales data, liquidity behaviour, order-flow imbalance, trade prints, bid/ask pressure, volume behaviour, and market microstructure signals — through a chatbot-style interface and a live research dashboard. + +It functions as a **disciplined institutional research desk with a risk officer beside it**, not as an uncontrolled autonomous trading bot. Every signal is probabilistic, every claim is bounded by available evidence, and no execution occurs without human authorisation. + +--- + +## Critical Compliance Constraint + +> **The system classifies behaviour, not identity.** + +The system may say: +- *"The activity resembles institutional-style accumulation."* +- *"The order flow suggests large-participant absorption."* +- *"Institutional-style accumulation probability: 68%, confidence: medium."* + +The system will **never** say: +- *"BlackRock is buying here."* +- *"Citadel is selling this level."* +- *"A specific hedge fund is accumulating."* + +Unless that claim is backed by legal, public, authorised, and verifiable data such as SEC filings or licensed institutional datasets. + +--- + +## Architecture + +``` +User / Chatbot Interface + ↓ +Agentic Orchestration Layer + ↓ +Market Data Intake Layer ← Polygon.io / Alpaca / IBKR / Sample feed + ↓ +Order Book Processing Engine + ↓ +Feature Engineering Layer + ↓ +Market Participant Behaviour Inference Layer + ↓ +[Phase 2] Machine Learning Prediction Layer + ↓ +Decision Parliament / Multi-Agent Review + ↓ +Risk & Compliance Guardrails + ↓ +Research Output / Alert / Paper Trade + ↓ +Audit Ledger + Explainability Report +``` + +--- + +## Phase 1 Components + +| Module | Location | Purpose | +|---|---|---| +| Sample Feed | `src/data_intake/sample_feed.py` | Synthetic L2 feed for offline development | +| Polygon Feed | `src/data_intake/polygon_feed.py` | Polygon.io WebSocket feed adapter | +| Data Integrity Agent | `src/data_intake/data_integrity_agent.py` | Feed quality, latency, bad-tick detection | +| Data Models | `src/data_intake/models.py` | Canonical Quote, OrderBookSnapshot, Trade models | +| Order Book Engine | `src/order_book/engine.py` | Depth, imbalance, absorption, sweep, spoof-like signals | +| Feature Engineer | `src/features/engineer.py` | 40+ ML-ready features (OB + technical + behavioral) | +| Order Book Analyst | `src/agents/order_book_analyst.py` | Directional bias from order book signals | +| Market-DNA Detector | `src/agents/market_dna_detector.py` | Regime classification | +| Institutional Footprint Agent | `src/agents/institutional_footprint.py` | Behavioural pattern inference (compliance-bounded) | +| Risk Governor | `src/agents/risk_governor.py` | Risk evaluation with veto authority | +| Compliance Agent | `src/agents/compliance_agent.py` | Hard-coded identity-inference blocks | +| Decision Parliament | `src/agents/decision_parliament.py` | Multi-agent voting and final disposition | +| Chatbot Interface | `src/chatbot/interface.py` | Natural-language command parsing and response | +| Audit Ledger | `src/audit/ledger.py` | Append-only JSONL audit log | +| Orchestrator | `src/orchestrator.py` | Full pipeline coordinator | +| Dashboard | `src/dashboard/app.py` | Streamlit research dashboard | +| CLI | `main.py` | Demo, chat, and dashboard launcher | + +--- + +## Quick Start + +### 1. Install dependencies + +```bash +pip install -r requirements.txt +``` + +### 2. Configure environment + +```bash +cp .env.example .env +# Edit .env — add your Polygon.io or Alpaca API keys if using live feeds. +# The sample feed works offline with no API keys required. +``` + +### 3. Run offline demo (no API keys needed) + +```bash +python main.py demo --symbol NVDA --ticks 30 --verbose +``` + +### 4. Interactive chatbot session + +```bash +python main.py chat --symbol AAPL +``` + +**Example commands in the chatbot:** +``` +analyze AAPL +is there buyer absorption? +are large sellers stacking the ask? +summarize institutional activity +what is the order flow regime? +explain the risk before entry +should this setup be paper-traded? +help +``` + +### 5. Launch the Streamlit dashboard + +```bash +python main.py dashboard +# or directly: +streamlit run src/dashboard/app.py +``` + +### 6. Run tests + +```bash +pytest tests/ -v +``` + +--- + +## Features Computed + +### Core Order Book Features (20+) +`bid_ask_imbalance`, `depth_weighted_imbalance`, `spread`, `mid_price`, `microprice`, `microprice_bias`, `order_flow_imbalance`, `absorption_score`, `sweep_intensity`, `spoof_like_score`, `iceberg_like_score`, `bid/ask_replenishment_rate`, `volume_at_bid/ask`, `rolling_buy_sell_ratio`, `spread_multiple`, `stacking_bid/ask`, `pulling_bid/ask`, … + +### Technical Indicators +`vwap`, `vwap_deviation`, `rvol`, `rsi_14`, `atr_14`, `macd_line/signal/histogram`, `bb_upper/lower/width/pct_b`, `obv`, `ma_slope_5/20` + +### Behavioural Composite Scores +`accumulation_score`, `distribution_score`, `institutional_footprint_prob`, `momentum_ignition_risk`, `liquidity_trap_risk`, `breakout_confirmation_score`, `false_breakout_prob` + +--- + +## Decision Parliament Dispositions + +| Disposition | Meaning | +|---|---| +| `research_approved` | Analysis delivered, no execution signal | +| `watchlist` | Setup identified, not yet actionable | +| `paper_trade_approved` | All agents clear — paper execution authorised | +| `human_review` | Conflicting signals — escalate to analyst | +| `rejected` | Evidence insufficient | +| `risk_veto` | Risk Governor blocked execution | +| `data_insufficient` | Feed quality too low for reliable output | +| `blocked` | Compliance Agent blocked the output | + +--- + +## Execution Modes + +| Stage | Status | +|---|---| +| 1. Research only | ✅ Active (Phase 1) | +| 2. Historical backtest | 🔲 Phase 2 | +| 3. Paper trading | 🔲 Phase 3 (broker sandbox) | +| 4. Simulated execution with slippage | 🔲 Phase 4 | +| 5. Human-approved tiny-capital live test | 🔲 Phase 5 | +| 6. Controlled monitored deployment | 🔲 Phase 6 | + +**Live trading is disabled by default.** `EXECUTION_MODE=paper` is enforced in configuration. `EXECUTION_MODE=live` requires explicit override, human approval, and passing all risk/compliance gates. + +--- + +## Development Roadmap + +| Phase | Description | Status | +|---|---|---| +| **Phase 1** | Research Prototype: L2 parser, feature engineering, chatbot, audit log, dashboard | ✅ Complete | +| **Phase 2** | ML Baseline: Random Forest / XGBoost, walk-forward validation, prediction dashboard | 🔲 Planned | +| **Phase 3** | Agentic Framework: LangGraph orchestration, News/Filings Agent, LLM explanation | 🔲 Planned | +| **Phase 4** | Deep Learning: LSTM, Transformer, DeepLOB, anomaly detection | 🔲 Planned | +| **Phase 5** | Paper Trading Execution: broker sandbox, slippage logging, RL-based execution | 🔲 Planned | +| **Phase 6** | Controlled Live Testing: tiny capital, auto-shutdown, compliance review | 🔲 Long-term | + +--- + +## Limitations + +**What Level 2 data can reasonably suggest:** +- Buyer/seller pressure at specific price levels +- Liquidity imbalance between bid and ask +- Absorption, accumulation-like, or distribution-like behaviour patterns +- Algorithmic or market-maker-like quoting patterns +- Spoof-like or iceberg-like activity signals + +**What Level 2 data cannot prove:** +- The legal identity of any specific buyer or seller +- Whether a named fund, institution, or individual is behind activity +- The true intent behind any order with certainty +- That any observed pattern will result in a specific price outcome + +--- + +## Technology Stack + +| Layer | Technology | +|---|---| +| Language | Python 3.11+ | +| Data Science | NumPy, pandas, Polars | +| ML (Phase 2+) | scikit-learn, XGBoost, LightGBM, PyTorch | +| Market Data | Polygon.io, Alpaca, Interactive Brokers API | +| Dashboard | Streamlit, Plotly | +| Agentic (Phase 3+) | LangGraph, CrewAI, or custom router | +| LLM (Phase 3+) | OpenAI / Anthropic / DeepSeek / Ollama (local) | +| Storage | PostgreSQL, Parquet, JSONL audit logs | +| API | FastAPI, WebSockets | + +--- + +## Responsible Use + +This system is a **research tool**. It does not constitute financial advice. Past performance of any signal or model does not guarantee future performance. All behavioural classifications are probabilistic estimates based on observable market data only. + +> The mature system behaves like a disciplined institutional research desk with a risk officer beside it — not like a reckless trading bot. diff --git a/main.py b/main.py new file mode 100644 index 0000000..e05d74e --- /dev/null +++ b/main.py @@ -0,0 +1,160 @@ +""" +main.py – CLI entry point for the Institutional Microstructure Intelligence System. + +Usage: + python main.py run --symbol NVDA # start live analysis loop + python main.py chat --symbol NVDA # interactive chatbot mode + python main.py demo --symbol AAPL --ticks 50 # offline demo with sample feed + python main.py dashboard # launch Streamlit dashboard + +Environment: + Copy .env.example → .env and configure API keys before running with live feeds. +""" + +from __future__ import annotations + +import asyncio +import subprocess +import sys + +import typer +from rich.console import Console +from rich.panel import Panel +from rich.text import Text + +app = typer.Typer( + name="microstructure", + help="Institutional Microstructure Intelligence System — Phase 1", + add_completion=False, +) +console = Console() + + +def _print_banner() -> None: + console.print(Panel( + Text.from_markup( + "[bold cyan]Institutional Microstructure Intelligence System[/bold cyan]\n" + "[dim]Phase 1 – Research Prototype | Paper-Only | Compliance-Aware[/dim]\n" + "[dim]Evidence first. Risk second. Execution last.[/dim]" + ), + border_style="cyan", + )) + + +@app.command() +def demo( + symbol: str = typer.Option("AAPL", "--symbol", "-s", help="Ticker symbol"), + ticks: int = typer.Option(20, "--ticks", "-t", help="Number of ticks to process"), + verbose: bool = typer.Option(False, "--verbose", "-v", help="Show full feature vector"), +) -> None: + """Run an offline demonstration using the sample (synthetic) feed.""" + _print_banner() + + from src.data_intake.sample_feed import SampleFeedAdapter + from src.orchestrator import Orchestrator + + console.print(f"\n[bold]Running demo for [cyan]{symbol}[/cyan] ({ticks} ticks)[/bold]\n") + + feed = SampleFeedAdapter(symbol=symbol, base_price=185.0, interval_ms=0) + orch = Orchestrator(symbol) + + async def _run(): + tick = 0 + async for snapshot, trades in feed.stream(max_ticks=ticks): + result = orch.process(snapshot, trades) + state = orch.latest_state + features = orch.latest_features + + disp_colour = { + "paper_trade_approved": "green", + "watchlist": "yellow", + "research_approved": "blue", + "risk_veto": "red", + "data_insufficient": "yellow", + "blocked": "red", + "human_review": "orange1", + "rejected": "dim", + }.get(result.disposition, "white") + + console.print( + f"[dim]Tick {tick+1:3d}[/dim] | " + f"Mid [bold]${state.mid_price:.2f}[/bold] | " + f"OBI [{'+' if state.book_imbalance >= 0 else ''}{state.book_imbalance:.3f}] | " + f"Spread ×{state.spread_multiple:.1f} | " + f"Regime [italic]{result.dna_regime}[/italic] | " + f"Footprint [italic]{result.inst_footprint_label}[/italic] ({result.inst_footprint_prob:.0%}) | " + f"Disposition [{disp_colour}]{result.disposition.replace('_', ' ').upper()}[/{disp_colour}]" + ) + if verbose and features: + console.print( + f" Absorption={features.absorption_score:.2f} " + f"Accum={features.accumulation_score:.2f} " + f"Distrib={features.distribution_score:.2f} " + f"InstProb={features.institutional_footprint_prob:.2f} " + f"RSI={features.rsi_14:.1f}" + ) + tick += 1 + + orch.close() + console.print(f"\n[dim]Audit log written to: {__import__('src.config', fromlist=['AUDIT_LOG_DIR']).AUDIT_LOG_DIR}[/dim]") + + asyncio.run(_run()) + + +@app.command() +def chat( + symbol: str = typer.Option("AAPL", "--symbol", "-s", help="Ticker symbol"), +) -> None: + """Interactive chatbot research session (offline, sample feed).""" + _print_banner() + + from src.data_intake.sample_feed import SampleFeedAdapter + from src.orchestrator import Orchestrator + + feed = SampleFeedAdapter(symbol=symbol, base_price=185.0) + orch = Orchestrator(symbol) + + # Warm up with 5 ticks so the pipeline has context + async def _warmup(): + tick = 0 + async for snap, trades in feed.stream(max_ticks=5): + orch.process(snap, trades) + tick += 1 + + asyncio.run(_warmup()) + + console.print( + f"\n[bold green]Chat session started for [cyan]{symbol}[/cyan].[/bold green] " + "Type [bold]'exit'[/bold] or [bold]'quit'[/bold] to end.\n" + ) + + while True: + try: + user_input = input("You > ").strip() + except (EOFError, KeyboardInterrupt): + break + if user_input.lower() in {"exit", "quit", "q"}: + break + if not user_input: + continue + + response = orch.chat(user_input) + console.print(f"\n[bold cyan]System >[/bold cyan]\n{response.text}\n") + + orch.close() + console.print("\n[dim]Session ended. Audit log saved.[/dim]") + + +@app.command() +def dashboard() -> None: + """Launch the Streamlit research dashboard.""" + _print_banner() + console.print("\n[bold]Launching Streamlit dashboard…[/bold]") + subprocess.run( + [sys.executable, "-m", "streamlit", "run", "src/dashboard/app.py"], + check=True, + ) + + +if __name__ == "__main__": + app() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ade29be --- /dev/null +++ b/requirements.txt @@ -0,0 +1,63 @@ +# ============================================================ +# Institutional Microstructure Intelligence System +# Phase 1 – Research Prototype Dependencies +# ============================================================ + +# Core data science +numpy>=1.26.0 +pandas>=2.1.0 +polars>=0.20.0 + +# Machine learning (baseline models – Phase 2 will expand) +scikit-learn>=1.4.0 +xgboost>=2.0.0 +lightgbm>=4.2.0 +shap>=0.44.0 + +# Market data connectors +polygon-api-client>=1.13.0 +alpaca-py>=0.20.0 +requests>=2.31.0 +websocket-client>=1.7.0 +aiohttp>=3.9.0 + +# API framework +fastapi>=0.110.0 +uvicorn[standard]>=0.27.0 +pydantic>=2.6.0 + +# Async & streaming +websockets>=12.0 +asyncio-mqtt>=0.16.0 + +# Dashboard & visualization +streamlit>=1.31.0 +plotly>=5.18.0 + +# LLM / Chatbot (local or API) +openai>=1.12.0 # optional – for GPT-based explanation +anthropic>=0.18.0 # optional – for Claude-based explanation +langchain>=0.1.0 +langchain-openai>=0.0.5 + +# Storage +sqlalchemy>=2.0.0 +psycopg2-binary>=2.9.9 # PostgreSQL driver +redis>=5.0.0 + +# Technical indicators +ta>=0.11.0 +pandas-ta>=0.3.14b + +# Utilities +python-dotenv>=1.0.0 +rich>=13.7.0 +typer>=0.9.0 +loguru>=0.7.2 +pyarrow>=15.0.0 # Parquet support +orjson>=3.9.0 # Fast JSON serialization + +# Testing +pytest>=8.0.0 +pytest-asyncio>=0.23.0 +pytest-cov>=4.1.0 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..447bc7a --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,9 @@ +""" +Institutional Microstructure Intelligence System +Phase 1 – Research Prototype + +Package root. Exposes top-level version and execution mode enforcement. +""" + +__version__ = "0.1.0" +__phase__ = "Phase 1 – Research Prototype" diff --git a/src/agents/__init__.py b/src/agents/__init__.py new file mode 100644 index 0000000..232b6e9 --- /dev/null +++ b/src/agents/__init__.py @@ -0,0 +1,3 @@ +""" +agents/__init__.py +""" diff --git a/src/agents/compliance_agent.py b/src/agents/compliance_agent.py new file mode 100644 index 0000000..4cce5ac --- /dev/null +++ b/src/agents/compliance_agent.py @@ -0,0 +1,142 @@ +""" +agents/compliance_agent.py – Compliance Agent (Phase 1). + +Hard-coded compliance enforcement layer. + +Responsibilities: + - Block identity-inference claims (HARD BLOCK) + - Detect prohibited language before any output is delivered + - Ensure limitations statement is present on all behavioral outputs + - Flag unlicensed data usage concerns + - Enforce paper-trading-only default + +This agent runs as the final gate before any output reaches the user. +A 'blocked' status prevents the response from being delivered. +""" + +from __future__ import annotations + +import re + +from src.config import IDENTITY_CLAIM_BLOCKED_TERMS, COMPLIANCE_LIMITATIONS_STATEMENT +from src.agents.models import ComplianceReport + + +class ComplianceAgent: + """ + Final compliance gate. + + Call check_output() on any text before it is delivered to the user. + Call check_behavioral_label() to validate behavioral inference outputs. + """ + + PROHIBITED_PATTERNS = [ + # Direct identity claims + r"\bis buying\b", + r"\bis selling\b", + r"\bis accumulating\b", + r"\bis distributing\b", + r"\bis entering\b", + r"\bis exiting\b", + r"behind this order", + r"behind the order", + r"behind this move", + r"behind this trade", + r"is the buyer", + r"is the seller", + r"is the institution", + r"named institution", + r"specific institution", + r"specific fund", + r"specific hedge fund", + r"specific individual", + r"specific person", + r"specific corporation", + ] + + def check_output(self, symbol: str, text: str) -> ComplianceReport: + """ + Scan generated text for compliance violations before delivery. + Returns ComplianceReport with status, violations, and warnings. + """ + violations: list[str] = [] + warnings: list[str] = [] + text_lower = text.lower() + + # ── Hard block: named institution claims ────────────────────────────── + for term in IDENTITY_CLAIM_BLOCKED_TERMS: + if term in text_lower: + violations.append( + f"Prohibited identity claim: text contains '{term}'. " + "Identity inference from Level 2 data is not permitted." + ) + + # ── Hard block: prohibited pattern phrases ──────────────────────────── + for pattern in self.PROHIBITED_PATTERNS: + if re.search(pattern, text_lower): + violations.append( + f"Prohibited claim pattern detected: '{pattern}'. " + "This language implies specific participant identity without authorised evidence." + ) + + # ── Warning: limitations statement missing ──────────────────────────── + if "level 2 data does not reveal" not in text_lower and "behavioral" in text_lower: + warnings.append( + "Behavioral classification output may be missing the required limitations statement." + ) + + # ── Warning: certainty language on probabilistic outputs ────────────── + certainty_terms = ["definitely", "certainly", "confirmed", "guaranteed", "proven"] + for ct in certainty_terms: + if ct in text_lower: + warnings.append( + f"Certainty language detected ('{ct}') in probabilistic output. " + "Consider replacing with confidence-qualified language." + ) + + if violations: + status = "blocked" + elif warnings: + status = "flagged" + else: + status = "approved" + + return ComplianceReport( + symbol=symbol, + timestamp_ns=__import__("time").time_ns(), + status=status, + violations=violations, + warnings=warnings, + ) + + def check_behavioral_label( + self, label: str, reasoning: list[str] + ) -> tuple[bool, str]: + """ + Validate that a behavioral label is compliant. + Returns (is_valid, error_message). + """ + permitted_labels = { + "accumulation_like", + "distribution_like", + "neutral", + "mixed", + "retail_like", + "algorithmic_like", + "market_maker_like", + "momentum_like", + "liquidity_seeking", + } + if label not in permitted_labels: + return False, ( + f"Behavioral label '{label}' is not in the permitted set. " + "Labels must describe observable behavior patterns, not participant identity." + ) + return True, "" + + @staticmethod + def add_limitations(text: str) -> str: + """Append the compliance limitations statement to any behavioral output.""" + if "level 2 data does not reveal" not in text.lower(): + return text + f"\n\n⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}" + return text diff --git a/src/agents/decision_parliament.py b/src/agents/decision_parliament.py new file mode 100644 index 0000000..60848a5 --- /dev/null +++ b/src/agents/decision_parliament.py @@ -0,0 +1,176 @@ +""" +agents/decision_parliament.py – Decision Parliament (Phase 1). + +Collects all agent assessments and produces a final disposition. + +Disposition hierarchy: + data_insufficient → (always first if data quality fails) + risk_veto → (Risk Governor veto) + blocked → (Compliance violation) + human_review → (conflicting high-confidence signals) + paper_trade_approved → (all clear, sufficient evidence) + watchlist → (soft signals, insufficient conviction) + research_approved → (analysis only, no execution context) + rejected → (no actionable signal) +""" + +from __future__ import annotations + +from src.agents.models import ( + ComplianceReport, + DecisionParliamentResult, + InstitutionalFootprintReport, + MarketDNAReport, + OrderBookAnalystReport, + RiskGovernorReport, +) +from src.config import COMPLIANCE_LIMITATIONS_STATEMENT + + +class DecisionParliament: + """ + Multi-agent voting and disposition engine. + """ + + def deliberate( + self, + ob_report: OrderBookAnalystReport, + dna_report: MarketDNAReport, + inst_report: InstitutionalFootprintReport, + risk_report: RiskGovernorReport, + compliance_report: ComplianceReport, + ) -> DecisionParliamentResult: + + symbol = ob_report.symbol + ts = ob_report.timestamp_ns + + # ── Gate 1: Data quality ────────────────────────────────────────────── + if not risk_report.data_quality_ok: + return self._result( + symbol, ts, "data_insufficient", + "Data quality check failed. Feed reliability is insufficient for analysis.", + ob_report, dna_report, inst_report, risk_report, compliance_report, + ) + + # ── Gate 2: Compliance block ────────────────────────────────────────── + if compliance_report.status == "blocked": + return self._result( + symbol, ts, "blocked", + f"Output blocked by Compliance Agent: {'; '.join(compliance_report.violations)}", + ob_report, dna_report, inst_report, risk_report, compliance_report, + ) + + # ── Gate 3: Risk veto ───────────────────────────────────────────────── + if risk_report.veto: + return self._result( + symbol, ts, "risk_veto", + f"Risk Governor veto: {'; '.join(risk_report.veto_reasons)}", + ob_report, dna_report, inst_report, risk_report, compliance_report, + ) + + # ── Compute aggregate conviction ────────────────────────────────────── + bullish_votes = 0 + bearish_votes = 0 + total_confidence = 0.0 + + if ob_report.directional_bias == "bullish": + bullish_votes += 1 + total_confidence += ob_report.bias_confidence + elif ob_report.directional_bias == "bearish": + bearish_votes += 1 + total_confidence += ob_report.bias_confidence + + if inst_report.behavioral_label == "accumulation_like": + bullish_votes += 1 + total_confidence += inst_report.probability + elif inst_report.behavioral_label == "distribution_like": + bearish_votes += 1 + total_confidence += inst_report.probability + + regime_supports_action = dna_report.regime in { + "accumulation", "breakout", "compression", "distribution" + } + if regime_supports_action: + total_confidence += dna_report.regime_confidence * 0.5 + + avg_confidence = total_confidence / max(bullish_votes + bearish_votes, 1) + + # ── Gate 4: Conflicting signals at high confidence ──────────────────── + if bullish_votes >= 1 and bearish_votes >= 1 and avg_confidence > 0.55: + return self._result( + symbol, ts, "human_review", + "Conflicting high-confidence signals from Order Book Analyst and " + "Institutional Footprint Agent. Human review recommended.", + ob_report, dna_report, inst_report, risk_report, compliance_report, + ) + + # ── Gate 5: Paper trade approval ───────────────────────────────────── + dominant_votes = max(bullish_votes, bearish_votes) + if ( + dominant_votes >= 2 + and avg_confidence >= 0.55 + and risk_report.risk_status == "clear" + and regime_supports_action + and compliance_report.status == "approved" + ): + direction = "bullish" if bullish_votes > bearish_votes else "bearish" + return self._result( + symbol, ts, "paper_trade_approved", + f"Multiple agents converge on {direction} view with sufficient confidence. " + f"Risk Governor clear. Paper trade approved (no live execution).", + ob_report, dna_report, inst_report, risk_report, compliance_report, + ) + + # ── Gate 6: Watchlist ───────────────────────────────────────────────── + if dominant_votes >= 1 and avg_confidence >= 0.40: + return self._result( + symbol, ts, "watchlist", + "Signal is present but conviction is insufficient for execution. " + "Setup added to watchlist for further monitoring.", + ob_report, dna_report, inst_report, risk_report, compliance_report, + ) + + # ── Default: Research only ──────────────────────────────────────────── + return self._result( + symbol, ts, "research_approved", + "Analysis complete. No actionable trade setup detected at this time. " + "Research output delivered.", + ob_report, dna_report, inst_report, risk_report, compliance_report, + ) + + @staticmethod + def _result( + symbol: str, + ts: int, + disposition: str, + reasoning: str, + ob: OrderBookAnalystReport, + dna: MarketDNAReport, + inst: InstitutionalFootprintReport, + risk: RiskGovernorReport, + compliance: ComplianceReport, + ) -> DecisionParliamentResult: + human_exp = ( + f"Symbol: {symbol} | Regime: {dna.regime} (confidence {dna.regime_confidence:.0%}) | " + f"Order book bias: {ob.directional_bias} ({ob.bias_confidence:.0%}) | " + f"Behavioral pattern: {inst.behavioral_label} ({inst.probability:.0%}, {inst.confidence_label} confidence) | " + f"Risk status: {risk.risk_status} | " + f"Disposition: {disposition.replace('_', ' ').upper()}. " + f"{reasoning}" + ) + return DecisionParliamentResult( + symbol=symbol, + timestamp_ns=ts, + disposition=disposition, + reasoning=reasoning, + ob_analyst_vote=ob.directional_bias, + ob_analyst_confidence=ob.bias_confidence, + dna_regime=dna.regime, + dna_confidence=dna.regime_confidence, + inst_footprint_label=inst.behavioral_label, + inst_footprint_prob=inst.probability, + risk_status=risk.risk_status, + compliance_status=compliance.status, + human_explanation=human_exp, + limitations=COMPLIANCE_LIMITATIONS_STATEMENT, + ) diff --git a/src/agents/institutional_footprint.py b/src/agents/institutional_footprint.py new file mode 100644 index 0000000..7491dda --- /dev/null +++ b/src/agents/institutional_footprint.py @@ -0,0 +1,112 @@ +""" +agents/institutional_footprint.py – Institutional Footprint Agent (Phase 1). + +Estimates broad behavioral patterns in the order flow. + +COMPLIANCE REQUIREMENT (HARD): + This agent NEVER identifies, names, or implies the identity of any + specific market participant, institution, firm, fund, or individual. + All outputs are probabilistic behavioral labels with mandatory + limitations statements. + +Permitted outputs (examples): + "The activity resembles institutional-style accumulation." + "The order flow suggests large-participant absorption." + "Institutional-style accumulation probability: 68%, confidence: medium." + +Prohibited outputs: + "BlackRock is buying here." + "Citadel is selling this level." + "A specific hedge fund is accumulating." +""" + +from __future__ import annotations + +from src.config import COMPLIANCE_LIMITATIONS_STATEMENT +from src.features.engineer import FeatureVector +from src.order_book.engine import OrderBookState +from src.agents.models import InstitutionalFootprintReport + + +class InstitutionalFootprintAgent: + + def analyse( + self, + state: OrderBookState, + features: FeatureVector, + ) -> InstitutionalFootprintReport: + label, prob, conf_label, reasoning = self._classify(state, features) + return InstitutionalFootprintReport( + symbol=state.symbol, + timestamp_ns=state.timestamp_ns, + behavioral_label=label, + probability=prob, + confidence_label=conf_label, + reasoning=reasoning, + limitations=COMPLIANCE_LIMITATIONS_STATEMENT, + ) + + @staticmethod + def _classify( + state: OrderBookState, fv: FeatureVector + ) -> tuple[str, float, str, list[str]]: + reasoning: list[str] = [] + accum_prob = fv.accumulation_score + distrib_prob = fv.distribution_score + + # Adjust probabilities with corroborating signals + if state.absorption_score > 0.5: + accum_prob += 0.08 + reasoning.append(f"Large-print absorption score: {state.absorption_score:.2f}.") + if state.bid_replenishment_rate > 0.3: + accum_prob += 0.06 + reasoning.append(f"Bid replenishment rate: {state.bid_replenishment_rate:.2f}.") + if state.iceberg_like_score > 0.4: + accum_prob += 0.06 + distrib_prob += 0.04 + reasoning.append(f"Iceberg-like clip patterns detected (score {state.iceberg_like_score:.2f}).") + if fv.order_flow_imbalance > 0.2: + accum_prob += 0.07 + reasoning.append(f"Positive order-flow imbalance: {fv.order_flow_imbalance:.2f}.") + elif fv.order_flow_imbalance < -0.2: + distrib_prob += 0.07 + reasoning.append(f"Negative order-flow imbalance: {fv.order_flow_imbalance:.2f}.") + if state.stacking_ask: + distrib_prob += 0.10 + reasoning.append("Ask-side stacking detected near resistance.") + if fv.vwap_deviation < -0.005: + distrib_prob += 0.05 + reasoning.append("Price trading below VWAP.") + elif fv.vwap_deviation > 0.005: + accum_prob += 0.05 + reasoning.append("Price trading above VWAP.") + + accum_prob = round(min(accum_prob, 1.0), 3) + distrib_prob = round(min(distrib_prob, 1.0), 3) + + if accum_prob > distrib_prob and accum_prob >= 0.40: + label = "accumulation_like" + prob = accum_prob + elif distrib_prob > accum_prob and distrib_prob >= 0.40: + label = "distribution_like" + prob = distrib_prob + elif max(accum_prob, distrib_prob) >= 0.25: + label = "mixed" + prob = max(accum_prob, distrib_prob) + else: + label = "neutral" + prob = max(accum_prob, distrib_prob) + + if prob >= 0.65: + conf = "high" + elif prob >= 0.45: + conf = "medium" + elif prob >= 0.30: + conf = "low" + else: + conf = "insufficient_data" + + if not reasoning: + reasoning.append("No strong corroborating behavioral signals detected.") + + return label, prob, conf, reasoning diff --git a/src/agents/market_dna_detector.py b/src/agents/market_dna_detector.py new file mode 100644 index 0000000..e844b12 --- /dev/null +++ b/src/agents/market_dna_detector.py @@ -0,0 +1,133 @@ +""" +agents/market_dna_detector.py – Market-DNA Detector Agent (Phase 1). + +Classifies the current market regime from order book and feature data. + +Regimes: + trending_up | trending_down | ranging | breakout | reversal | + compression | expansion | accumulation | distribution | trap | ambiguous +""" + +from __future__ import annotations + +import math + +from src.features.engineer import FeatureVector +from src.order_book.engine import OrderBookState +from src.agents.models import MarketDNAReport + + +class MarketDNADetectorAgent: + """ + Rule-based regime classifier for Phase 1. + Phase 4 will replace the rule layer with an HMM / TCN model. + """ + + def classify( + self, + state: OrderBookState, + features: FeatureVector, + ) -> MarketDNAReport: + regime, confidence, signals, stability = self._classify_regime(state, features) + return MarketDNAReport( + symbol=state.symbol, + timestamp_ns=state.timestamp_ns, + regime=regime, + regime_confidence=confidence, + supporting_signals=signals, + regime_stability=stability, + ) + + @staticmethod + def _classify_regime( + state: OrderBookState, fv: FeatureVector + ) -> tuple[str, float, list[str], float]: + signals: list[str] = [] + scores: dict[str, float] = {} + + # ── Compression ─────────────────────────────────────────────────────── + compression = 0.0 + if not math.isnan(fv.bb_width) and fv.bb_width < 0.5: + compression += 0.4 + signals.append("Bollinger Band width compressed.") + if state.spread_multiple < 1.2: + compression += 0.2 + signals.append("Spread near historical average.") + scores["compression"] = compression + + # ── Breakout ────────────────────────────────────────────────────────── + breakout = 0.0 + if fv.sweep_intensity > 0.5: + breakout += 0.35 + signals.append(f"Sweep intensity elevated ({fv.sweep_intensity:.2f}).") + if fv.rvol > 1.8: + breakout += 0.30 + signals.append(f"Relative volume elevated ({fv.rvol:.1f}x).") + if fv.breakout_confirmation_score > 0.5: + breakout += 0.35 + scores["breakout"] = breakout + + # ── Accumulation ────────────────────────────────────────────────────── + accum = 0.0 + if fv.accumulation_score > 0.5: + accum += 0.5 + signals.append(f"Accumulation score {fv.accumulation_score:.2f}.") + if state.absorption_score > 0.5: + accum += 0.3 + if state.bid_replenishment_rate > 0.3: + accum += 0.2 + scores["accumulation"] = accum + + # ── Distribution ────────────────────────────────────────────────────── + distrib = 0.0 + if fv.distribution_score > 0.5: + distrib += 0.5 + signals.append(f"Distribution score {fv.distribution_score:.2f}.") + if state.stacking_ask: + distrib += 0.3 + scores["distribution"] = distrib + + # ── Ranging ─────────────────────────────────────────────────────────── + ranging = 0.0 + if abs(state.book_imbalance) < 0.15 and state.sweep_intensity < 0.3: + ranging += 0.5 + signals.append("Low imbalance and low sweep: ranging market.") + if fv.rvol < 0.8: + ranging += 0.3 + scores["ranging"] = ranging + + # ── Trap ───────────────────────────────────────────────────────────── + trap = 0.0 + if fv.liquidity_trap_risk > 0.5: + trap += 0.5 + signals.append(f"Liquidity trap risk {fv.liquidity_trap_risk:.2f}.") + if fv.false_breakout_prob > 0.4: + trap += 0.3 + scores["trap"] = trap + + # ── Expansion (post-breakout volatility) ───────────────────────────── + expansion = 0.0 + if not math.isnan(fv.bb_width) and fv.bb_width > 1.5: + expansion += 0.4 + if not math.isnan(fv.atr_14) and fv.atr_14 > 0: + expansion += 0.3 + scores["expansion"] = expansion + + # ── Select dominant regime ──────────────────────────────────────────── + if not scores: + return "ambiguous", 0.3, signals, 0.3 + + best_regime = max(scores, key=lambda k: scores[k]) + best_score = scores[best_regime] + + if best_score < 0.3: + regime = "ambiguous" + confidence = 0.3 + else: + regime = best_regime + confidence = round(min(best_score, 1.0), 3) + + stability = round(1.0 - (len([s for s in scores.values() if s > 0.3]) - 1) * 0.1, 3) + stability = max(0.0, stability) + + return regime, confidence, signals[:5], stability diff --git a/src/agents/models.py b/src/agents/models.py new file mode 100644 index 0000000..5a97957 --- /dev/null +++ b/src/agents/models.py @@ -0,0 +1,92 @@ +""" +agents/models.py – Agent output data models. + +All agent assessments are typed dataclasses. The Decision Parliament +collects these and computes the final disposition. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class OrderBookAnalystReport: + symbol: str + timestamp_ns: int + directional_bias: str # "bullish" | "bearish" | "neutral" + bias_confidence: float # 0.0 – 1.0 + key_signals: list[str] + absorption_score: float + sweep_intensity: float + spoof_like_score: float + iceberg_like_score: float + spread_multiple: float + book_imbalance: float + + +@dataclass +class MarketDNAReport: + symbol: str + timestamp_ns: int + regime: str # trend | range | breakout | reversal | compression | + # expansion | accumulation | distribution | trap | ambiguous + regime_confidence: float + supporting_signals: list[str] + regime_stability: float # 0.0 – 1.0 + + +@dataclass +class InstitutionalFootprintReport: + symbol: str + timestamp_ns: int + behavioral_label: str # accumulation_like | distribution_like | neutral | mixed + probability: float + confidence_label: str # low | medium | high | insufficient_data + reasoning: list[str] + limitations: str # always populated – compliance requirement + + +@dataclass +class RiskGovernorReport: + symbol: str + timestamp_ns: int + risk_status: str # clear | caution | veto + risk_score: float # 0.0 – 1.0 (higher = more risk) + veto: bool + veto_reasons: list[str] + spread_ok: bool + data_quality_ok: bool + model_confidence_ok: bool + news_lockout: bool + + +@dataclass +class ComplianceReport: + symbol: str + timestamp_ns: int + status: str # approved | flagged | blocked + violations: list[str] + warnings: list[str] + + +@dataclass +class DecisionParliamentResult: + symbol: str + timestamp_ns: int + disposition: str # research_approved | watchlist | paper_trade_approved | + # human_review | rejected | risk_veto | data_insufficient + reasoning: str + ob_analyst_vote: str + ob_analyst_confidence: float + dna_regime: str + dna_confidence: float + inst_footprint_label: str + inst_footprint_prob: float + risk_status: str + compliance_status: str + human_explanation: str + limitations: str + timestamp_ms: int = field(default_factory=lambda: int(time.time() * 1000)) diff --git a/src/agents/order_book_analyst.py b/src/agents/order_book_analyst.py new file mode 100644 index 0000000..f8d03b8 --- /dev/null +++ b/src/agents/order_book_analyst.py @@ -0,0 +1,92 @@ +""" +agents/order_book_analyst.py – Order Book Analyst Agent (Phase 1). + +Interprets the fully processed OrderBookState and produces a structured +directional assessment with supporting signals. +""" + +from __future__ import annotations + +from src.features.engineer import FeatureVector +from src.order_book.engine import OrderBookState +from src.agents.models import OrderBookAnalystReport + + +class OrderBookAnalystAgent: + """ + Analyses bid/ask pressure, imbalance, stacking, pulling, absorption, + sweep activity, spread behaviour, and microprice pressure. + + Returns a structured OrderBookAnalystReport. + """ + + def analyse( + self, + state: OrderBookState, + features: FeatureVector, + ) -> OrderBookAnalystReport: + bias, confidence = self._compute_bias(features) + return OrderBookAnalystReport( + symbol=state.symbol, + timestamp_ns=state.timestamp_ns, + directional_bias=bias, + bias_confidence=confidence, + key_signals=state.signals, + absorption_score=state.absorption_score, + sweep_intensity=state.sweep_intensity, + spoof_like_score=state.spoof_like_score, + iceberg_like_score=state.iceberg_like_score, + spread_multiple=state.spread_multiple, + book_imbalance=state.book_imbalance, + ) + + @staticmethod + def _compute_bias(fv: FeatureVector) -> tuple[str, float]: + """ + Combine multiple signals into a directional bias score. + Returns (direction, confidence). + """ + score = 0.0 + + # Order flow imbalance (primary signal) + score += fv.order_flow_imbalance * 0.30 + + # Book imbalance + score += fv.bid_ask_imbalance * 0.20 + + # Microprice bias + score += fv.microprice_bias * 200 * 0.10 # normalise pips + + # Accumulation vs distribution + score += (fv.accumulation_score - fv.distribution_score) * 0.20 + + # Buy/sell ratio (centralised around 0.5) + score += (fv.rolling_buy_sell_ratio - 0.5) * 2 * 0.10 + + # Absorption increases conviction in the dominant side + if fv.order_flow_imbalance > 0 and fv.absorption_score > 0.5: + score += 0.10 + elif fv.order_flow_imbalance < 0 and fv.absorption_score > 0.5: + score -= 0.10 + + # Stacking on ask = bearish signal + if fv.stacking_ask: + score -= 0.10 + if fv.stacking_bid: + score += 0.10 + + # Pulling on bid = bearish signal + if fv.pulling_bid: + score -= 0.10 + if fv.pulling_ask: + score += 0.10 + + if score > 0.15: + direction = "bullish" + elif score < -0.15: + direction = "bearish" + else: + direction = "neutral" + + confidence = round(min(abs(score), 1.0), 3) + return direction, confidence diff --git a/src/agents/risk_governor.py b/src/agents/risk_governor.py new file mode 100644 index 0000000..e0e77d8 --- /dev/null +++ b/src/agents/risk_governor.py @@ -0,0 +1,124 @@ +""" +agents/risk_governor.py – Risk Governor Agent / Captain MarginCall (Phase 1). + +Evaluates every setup for risk acceptability. +Has veto authority: a Veto status blocks all execution recommendations. + +Evaluates: + - Spread vs historical average + - Data quality score + - Model confidence (placeholder in Phase 1) + - Spread multiple + - Liquidity trap risk + - Momentum ignition risk + - Spoof-like conditions + - News lockout (hardcoded False in Phase 1 — Phase 3 adds News Agent) +""" + +from __future__ import annotations + +from src.config import ( + MAX_SPREAD_MULTIPLE, + MIN_DATA_QUALITY_SCORE, + MIN_MODEL_CONFIDENCE, +) +from src.data_intake.models import FeedHealthReport +from src.features.engineer import FeatureVector +from src.order_book.engine import OrderBookState +from src.agents.models import RiskGovernorReport + + +class RiskGovernorAgent: + """ + The system's risk officer. Produces a RiskGovernorReport and can veto + any execution recommendation. + """ + + def evaluate( + self, + state: OrderBookState, + features: FeatureVector, + health: FeedHealthReport, + model_confidence: float = float("nan"), + ) -> RiskGovernorReport: + veto_reasons: list[str] = [] + + # ── Spread check ────────────────────────────────────────────────────── + spread_ok = state.spread_multiple <= MAX_SPREAD_MULTIPLE + if not spread_ok: + veto_reasons.append( + f"Spread is {state.spread_multiple:.1f}× the 20-snapshot average " + f"(threshold: {MAX_SPREAD_MULTIPLE}×). Execution risk elevated." + ) + + # ── Data quality check ──────────────────────────────────────────────── + dq_ok = health.quality_score >= MIN_DATA_QUALITY_SCORE + if not dq_ok: + veto_reasons.append( + f"Data quality score {health.quality_score:.2f} is below threshold " + f"{MIN_DATA_QUALITY_SCORE}." + ) + if health.stale_quote: + veto_reasons.append("Feed has a stale quote — data freshness cannot be guaranteed.") + if health.gap_detected: + veto_reasons.append("Sequence gap detected in feed — order book state may be incomplete.") + + # ── Model confidence ────────────────────────────────────────────────── + import math + conf_ok = math.isnan(model_confidence) or model_confidence >= MIN_MODEL_CONFIDENCE + if not conf_ok: + veto_reasons.append( + f"ML model confidence {model_confidence:.2f} below threshold " + f"{MIN_MODEL_CONFIDENCE}. Signal reliability is insufficient." + ) + + # ── Liquidity trap ──────────────────────────────────────────────────── + if features.liquidity_trap_risk > 0.6: + veto_reasons.append( + f"Liquidity trap risk elevated ({features.liquidity_trap_risk:.2f}). " + "Wide spread, thin depth, or rapid liquidity removal detected." + ) + + # ── Spoof-like conditions ───────────────────────────────────────────── + if features.spoof_like_score > 0.5: + veto_reasons.append( + f"Spoof-like behaviour score elevated ({features.spoof_like_score:.2f}). " + "Book may not reflect true supply/demand." + ) + + # ── News lockout (Phase 1: static False; Phase 3 adds live check) ───── + news_lockout = False + + # ── Risk score ──────────────────────────────────────────────────────── + risk_score = 0.0 + if not spread_ok: + risk_score += 0.30 + if not dq_ok: + risk_score += 0.30 + if not conf_ok: + risk_score += 0.15 + risk_score += features.liquidity_trap_risk * 0.15 + risk_score += features.spoof_like_score * 0.10 + risk_score = round(min(risk_score, 1.0), 3) + + # ── Status ──────────────────────────────────────────────────────────── + veto = len(veto_reasons) > 0 and risk_score >= 0.30 + if veto: + status = "veto" + elif risk_score >= 0.20: + status = "caution" + else: + status = "clear" + + return RiskGovernorReport( + symbol=state.symbol, + timestamp_ns=state.timestamp_ns, + risk_status=status, + risk_score=risk_score, + veto=veto, + veto_reasons=veto_reasons, + spread_ok=spread_ok, + data_quality_ok=dq_ok, + model_confidence_ok=conf_ok, + news_lockout=news_lockout, + ) diff --git a/src/audit/__init__.py b/src/audit/__init__.py new file mode 100644 index 0000000..549f270 --- /dev/null +++ b/src/audit/__init__.py @@ -0,0 +1,3 @@ +""" +audit/__init__.py +""" diff --git a/src/audit/ledger.py b/src/audit/ledger.py new file mode 100644 index 0000000..916d5ad --- /dev/null +++ b/src/audit/ledger.py @@ -0,0 +1,114 @@ +""" +audit/ledger.py – Append-only JSONL Audit Ledger (Phase 1). + +Every system decision, agent vote, feature snapshot, and risk assessment +is written to a date-partitioned JSONL file. + +Design principles: + - Append-only: records are never modified or deleted + - Every record is a self-contained JSON object on a single line + - Human-readable and machine-parseable + - Phase 2+ will add PostgreSQL/Parquet archival layer + +Record types: + decision – Decision Parliament result (written for every tick) + data_quality – FeedHealthReport + feature – FeatureVector snapshot (sampled, not every tick) +""" + +from __future__ import annotations + +import json +import time +from dataclasses import asdict +from pathlib import Path + +import orjson + +from src.config import AUDIT_LOG_DIR +from src.agents.models import DecisionParliamentResult +from src.data_intake.models import FeedHealthReport +from src.features.engineer import FeatureVector + + +class AuditLedger: + """ + Thread-safe append-only JSONL audit ledger. + + One file per symbol per date: e.g., NVDA_2026-06-17.jsonl + """ + + def __init__(self, log_dir: Path = AUDIT_LOG_DIR) -> None: + self._log_dir = log_dir + self._log_dir.mkdir(parents=True, exist_ok=True) + self._handles: dict[str, object] = {} + + # ── Public ──────────────────────────────────────────────────────────────── + + def log_decision(self, result: DecisionParliamentResult) -> None: + record = { + "record_type": "decision", + "timestamp": result.timestamp_ns, + "timestamp_ms": result.timestamp_ms, + "symbol": result.symbol, + "disposition": result.disposition, + "reasoning": result.reasoning, + "ob_analyst_vote": result.ob_analyst_vote, + "ob_analyst_confidence": result.ob_analyst_confidence, + "dna_regime": result.dna_regime, + "dna_confidence": result.dna_confidence, + "inst_footprint_label": result.inst_footprint_label, + "inst_footprint_prob": result.inst_footprint_prob, + "risk_status": result.risk_status, + "compliance_status": result.compliance_status, + "human_explanation": result.human_explanation, + "limitations": result.limitations, + } + self._write(result.symbol, record) + + def log_health(self, health: FeedHealthReport) -> None: + record = { + "record_type": "data_quality", + "timestamp": health.timestamp_ns, + "symbol": health.symbol, + "feed": health.feed, + "quality_score": health.quality_score, + "latency_ms": health.latency_ms, + "stale_quote": health.stale_quote, + "gap_detected": health.gap_detected, + "bad_ticks": health.bad_ticks, + "missing_ticks": health.missing_ticks, + "is_healthy": health.is_healthy, + "notes": health.notes, + } + self._write(health.symbol, record) + + def log_features(self, fv: FeatureVector) -> None: + record = {"record_type": "feature"} | fv.to_dict() + self._write(fv.symbol, record) + + def close(self) -> None: + for fh in self._handles.values(): + try: + fh.close() # type: ignore[attr-defined] + except Exception: + pass + + # ── Private ─────────────────────────────────────────────────────────────── + + def _write(self, symbol: str, record: dict) -> None: + fh = self._get_handle(symbol) + try: + line = orjson.dumps(record).decode() + "\n" + except Exception: + line = json.dumps(record, default=str) + "\n" + fh.write(line) # type: ignore[attr-defined] + fh.flush() # type: ignore[attr-defined] + + def _get_handle(self, symbol: str): + from datetime import date + key = f"{symbol}_{date.today().isoformat()}" + if key not in self._handles: + path = self._log_dir / f"{key}.jsonl" + self._handles[key] = open(path, "a", encoding="utf-8") + return self._handles[key] diff --git a/src/chatbot/__init__.py b/src/chatbot/__init__.py new file mode 100644 index 0000000..89b9f37 --- /dev/null +++ b/src/chatbot/__init__.py @@ -0,0 +1,3 @@ +""" +chatbot/__init__.py +""" diff --git a/src/chatbot/interface.py b/src/chatbot/interface.py new file mode 100644 index 0000000..c3c0f3f --- /dev/null +++ b/src/chatbot/interface.py @@ -0,0 +1,359 @@ +""" +chatbot/interface.py – Chatbot Interface Layer (Phase 1). + +The user-facing command centre. Accepts natural-language queries and routes +them to the appropriate agents. Returns structured, human-readable responses. + +Supported commands (Phase 1 — no live LLM required): + "analyze {SYMBOL}" + "analyze {SYMBOL} level 2" + "is there buyer absorption" + "are large sellers stacking the ask" + "summarize institutional activity" + "what is the order flow regime" + "should this be paper traded" + "explain the risk" + "show signals" + "help" + +Phase 3 will replace the rule-based intent parser with a local or API-based +LLM (OpenAI / Anthropic / DeepSeek) to support free-form natural-language. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Optional + +from src.agents.models import ( + ComplianceReport, + DecisionParliamentResult, + InstitutionalFootprintReport, + MarketDNAReport, + OrderBookAnalystReport, + RiskGovernorReport, +) +from src.config import COMPLIANCE_LIMITATIONS_STATEMENT + + +HELP_TEXT = """ +╔══════════════════════════════════════════════════════════════════╗ +║ Institutional Microstructure Intelligence — Phase 1 Chatbot ║ +╚══════════════════════════════════════════════════════════════════╝ + +Available commands: + analyze Full Level 2 microstructure analysis + absorption Is buyer absorption present at the bid? + stacking Are large sellers stacking the ask? + institutional Summarise institutional-style activity + regime What is the order-flow regime? + risk Explain the risk before entry + paper-trade Should this setup be paper-traded? + signals Show active order book signals + help Show this help text + +Notes: + • All analysis is for research purposes only. + • No live execution. Paper-trade mode only by default. + • Behavioural classifications are probabilistic estimates. + • Participant identity CANNOT be inferred from Level 2 data. +""".strip() + + +@dataclass +class ChatbotResponse: + text: str + symbol: Optional[str] + intent: str + compliance_status: str + disposition: Optional[str] = None + + +class ChatbotInterface: + """ + Rule-based intent parser and response formatter. + Phase 3 will swap the parser for an LLM-based router. + """ + + INTENTS: list[tuple[str, list[str]]] = [ + ("analyze", ["analyze", "analysis", "level 2", "l2", "activity"]), + ("absorption", ["absorption", "buyer absorption", "absorb"]), + ("stacking", ["stacking", "stack", "large sellers", "ask wall"]), + ("institutional", ["institutional", "large participant", "footprint", "accumulation", "distribution"]), + ("regime", ["regime", "order flow regime", "market dna", "market regime"]), + ("risk", ["risk", "explain risk", "before entry", "risk before"]), + ("paper_trade", ["paper trade", "paper-trade", "should i trade", "should this be traded"]), + ("signals", ["signals", "active signals", "show signals"]), + ("help", ["help", "commands", "usage", "?"]), + ] + + def parse_intent(self, user_input: str) -> tuple[str, Optional[str]]: + """Extract intent and optional symbol from raw user input.""" + text = user_input.lower().strip() + + # Extract ticker symbol: 1–5 uppercase letters, possibly followed/preceded by space + symbol_match = re.search(r'\b([A-Z]{1,5})\b', user_input) + symbol = symbol_match.group(1) if symbol_match else None + + for intent, keywords in self.INTENTS: + if any(kw in text for kw in keywords): + return intent, symbol + + return "unknown", symbol + + def respond( + self, + user_input: str, + ob_report: Optional[OrderBookAnalystReport] = None, + dna_report: Optional[MarketDNAReport] = None, + inst_report: Optional[InstitutionalFootprintReport] = None, + risk_report: Optional[RiskGovernorReport] = None, + compliance_report: Optional[ComplianceReport] = None, + parliament_result: Optional[DecisionParliamentResult] = None, + ) -> ChatbotResponse: + """ + Generate a human-readable response given user input and available agent reports. + """ + intent, symbol = self.parse_intent(user_input) + + if intent == "help" or not any([ob_report, dna_report, parliament_result]): + return ChatbotResponse( + text=HELP_TEXT, + symbol=symbol, + intent=intent, + compliance_status="approved", + ) + + if intent == "analyze": + text = self._fmt_analyze(symbol, ob_report, dna_report, inst_report, risk_report, parliament_result) + elif intent == "absorption": + text = self._fmt_absorption(symbol, ob_report) + elif intent == "stacking": + text = self._fmt_stacking(symbol, ob_report) + elif intent == "institutional": + text = self._fmt_institutional(symbol, inst_report) + elif intent == "regime": + text = self._fmt_regime(symbol, dna_report) + elif intent == "risk": + text = self._fmt_risk(symbol, risk_report) + elif intent == "paper_trade": + text = self._fmt_paper_trade(symbol, parliament_result, risk_report) + elif intent == "signals": + text = self._fmt_signals(symbol, ob_report) + else: + text = ( + f"I'm not sure how to interpret that command.\n" + f"Type 'help' for a list of supported commands." + ) + + comp_status = compliance_report.status if compliance_report else "approved" + if comp_status == "blocked": + text = ( + "⛔ This output was blocked by the Compliance Agent.\n" + + (compliance_report.violations[0] if compliance_report and compliance_report.violations else "") + ) + + return ChatbotResponse( + text=text, + symbol=symbol, + intent=intent, + compliance_status=comp_status, + disposition=parliament_result.disposition if parliament_result else None, + ) + + # ── Formatters ───────────────────────────────────────────────────────────── + + @staticmethod + def _fmt_analyze( + symbol, ob, dna, inst, risk, parl + ) -> str: + lines = [ + f"{'═'*60}", + f" MICROSTRUCTURE ANALYSIS: {symbol or 'N/A'}", + f"{'═'*60}", + ] + if dna: + lines += [ + f" Market Regime : {dna.regime.upper().replace('_', ' ')}", + f" Regime Confidence : {dna.regime_confidence:.0%}", + ] + if ob: + lines += [ + f" Order Book Bias : {ob.directional_bias.upper()}", + f" Bias Confidence : {ob.bias_confidence:.0%}", + f" Book Imbalance : {ob.book_imbalance:+.3f}", + f" Spread Multiple : {ob.spread_multiple:.1f}×", + f" Absorption Score : {ob.absorption_score:.2f}", + f" Sweep Intensity : {ob.sweep_intensity:.2f}", + f" Spoof-like Score : {ob.spoof_like_score:.2f}", + ] + if inst: + lines += [ + "", + f" Behavioral Pattern: {inst.behavioral_label.replace('_', ' ').upper()}", + f" Footprint Prob : {inst.probability:.0%} [{inst.confidence_label} confidence]", + f" Reasoning :", + ] + for r in inst.reasoning[:3]: + lines.append(f" • {r}") + if risk: + lines += [ + "", + f" Risk Status : {risk.risk_status.upper()}", + f" Risk Score : {risk.risk_score:.2f}", + ] + if risk.veto_reasons: + lines.append(" Risk Warnings :") + for v in risk.veto_reasons[:3]: + lines.append(f" ⚠ {v}") + if parl: + lines += [ + "", + f" Decision : {parl.disposition.replace('_', ' ').upper()}", + f" Reasoning : {parl.reasoning}", + ] + lines += [ + "", + f" ⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}", + f"{'─'*60}", + ] + return "\n".join(lines) + + @staticmethod + def _fmt_absorption(symbol, ob) -> str: + if not ob: + return "No order book data available." + score = ob.absorption_score + label = "elevated" if score > 0.6 else "moderate" if score > 0.3 else "low" + return ( + f"Buyer Absorption Analysis — {symbol or 'N/A'}\n" + f"{'─'*40}\n" + f"Absorption Score : {score:.2f} ({label})\n" + f"Book Imbalance : {ob.book_imbalance:+.3f}\n\n" + + ( + f"Large buy prints are being absorbed at or near the bid without sustained " + f"upward price continuation. This pattern is consistent with seller supply " + f"meeting buyer demand at the level." + if score > 0.5 else + "No significant buyer absorption detected at this time." + ) + + f"\n\n⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}" + ) + + @staticmethod + def _fmt_stacking(symbol, ob) -> str: + if not ob: + return "No order book data available." + stacking = "stacking_ask" in [s.lower() for s in ob.key_signals if "stacking" in s.lower()] + ask_signals = [s for s in ob.key_signals if "ask" in s.lower()] + return ( + f"Ask-Side Stacking Analysis — {symbol or 'N/A'}\n" + f"{'─'*40}\n" + f"Ask Stacking : {'DETECTED' if stacking else 'Not detected'}\n" + f"Spoof-like Score : {ob.spoof_like_score:.2f}\n" + f"Book Imbalance : {ob.book_imbalance:+.3f}\n\n" + + ("\n".join(ask_signals) if ask_signals else "No significant ask-side stacking detected.") + + f"\n\n⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}" + ) + + @staticmethod + def _fmt_institutional(symbol, inst) -> str: + if not inst: + return "No institutional footprint data available." + lines = [ + f"Institutional-Style Activity — {symbol or 'N/A'}", + f"{'─'*44}", + f"Behavioral Pattern : {inst.behavioral_label.replace('_', ' ').upper()}", + f"Probability : {inst.probability:.0%}", + f"Confidence : {inst.confidence_label.upper()}", + "", + "Reasoning:", + ] + for r in inst.reasoning: + lines.append(f" • {r}") + lines += ["", f"⚠️ {inst.limitations}"] + return "\n".join(lines) + + @staticmethod + def _fmt_regime(symbol, dna) -> str: + if not dna: + return "No market regime data available." + lines = [ + f"Market Regime — {symbol or 'N/A'}", + f"{'─'*36}", + f"Regime : {dna.regime.upper().replace('_', ' ')}", + f"Confidence : {dna.regime_confidence:.0%}", + f"Stability : {dna.regime_stability:.0%}", + "", + "Supporting signals:", + ] + for s in dna.supporting_signals: + lines.append(f" • {s}") + return "\n".join(lines) + + @staticmethod + def _fmt_risk(symbol, risk) -> str: + if not risk: + return "No risk assessment available." + lines = [ + f"Risk Assessment — {symbol or 'N/A'}", + f"{'─'*38}", + f"Status : {risk.risk_status.upper()}", + f"Risk Score : {risk.risk_score:.2f} (0=low, 1=high)", + f"Spread OK : {'✓' if risk.spread_ok else '✗'}", + f"Data OK : {'✓' if risk.data_quality_ok else '✗'}", + f"Model Conf. : {'✓' if risk.model_confidence_ok else '✗'}", + f"News Lockout : {'YES — trading paused' if risk.news_lockout else 'No active lockout'}", + ] + if risk.veto_reasons: + lines.append("\nRisk warnings:") + for v in risk.veto_reasons: + lines.append(f" ⚠ {v}") + lines.append("\nAll execution is paper-only by default. Human approval required for live trading.") + return "\n".join(lines) + + @staticmethod + def _fmt_paper_trade(symbol, parl, risk) -> str: + if not parl: + return "No Decision Parliament result available." + disp = parl.disposition + lines = [ + f"Paper Trade Assessment — {symbol or 'N/A'}", + f"{'─'*44}", + f"Disposition : {disp.replace('_', ' ').upper()}", + f"Reasoning : {parl.reasoning}", + "", + f"Agent Votes:", + f" Order Book Analyst : {parl.ob_analyst_vote.upper()} ({parl.ob_analyst_confidence:.0%})", + f" Market DNA : {parl.dna_regime.upper()} ({parl.dna_confidence:.0%})", + f" Inst. Footprint : {parl.inst_footprint_label.replace('_',' ').upper()} ({parl.inst_footprint_prob:.0%})", + f" Risk Governor : {parl.risk_status.upper()}", + f" Compliance : {parl.compliance_status.upper()}", + ] + if disp == "paper_trade_approved": + lines.append( + "\n✅ Setup approved for PAPER TRADE ONLY. " + "No live capital. Human review before any live execution." + ) + elif disp == "risk_veto": + if risk and risk.veto_reasons: + lines.append(f"\n⛔ Risk Veto: {risk.veto_reasons[0]}") + elif disp == "data_insufficient": + lines.append("\n⚠️ Data quality is insufficient. Analysis paused.") + else: + lines.append(f"\n⏸ Outcome: {disp.replace('_', ' ')}. No execution recommended.") + lines += ["", f"⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}"] + return "\n".join(lines) + + @staticmethod + def _fmt_signals(symbol, ob) -> str: + if not ob: + return "No active signals." + lines = [ + f"Active Order Book Signals — {symbol or 'N/A'}", + f"{'─'*44}", + ] + for s in ob.key_signals: + lines.append(f" • {s}") + return "\n".join(lines) diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..3fd6556 --- /dev/null +++ b/src/config.py @@ -0,0 +1,67 @@ +""" +config.py – Centralised system configuration loaded from environment variables. +All subsystems import from here; never read os.environ directly in business logic. +""" + +import os +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv() + +# ── Project layout ──────────────────────────────────────────────────────────── +ROOT_DIR = Path(__file__).resolve().parent.parent +AUDIT_LOG_DIR = ROOT_DIR / os.getenv("AUDIT_LOG_DIR", "data/audit_logs") +AUDIT_LOG_DIR.mkdir(parents=True, exist_ok=True) + +# ── Data feeds ──────────────────────────────────────────────────────────────── +POLYGON_API_KEY: str = os.getenv("POLYGON_API_KEY", "") +ALPACA_API_KEY: str = os.getenv("ALPACA_API_KEY", "") +ALPACA_SECRET_KEY: str = os.getenv("ALPACA_SECRET_KEY", "") +ALPACA_BASE_URL: str = os.getenv("ALPACA_BASE_URL", "https://paper-api.alpaca.markets") +DEFAULT_FEED: str = os.getenv("DEFAULT_FEED", "sample") +DEFAULT_SYMBOL: str = os.getenv("DEFAULT_SYMBOL", "AAPL") + +# ── LLM ─────────────────────────────────────────────────────────────────────── +OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "") +ANTHROPIC_API_KEY: str = os.getenv("ANTHROPIC_API_KEY", "") + +# ── Storage ─────────────────────────────────────────────────────────────────── +DATABASE_URL: str = os.getenv("DATABASE_URL", "sqlite:///data/microstructure.db") +REDIS_URL: str = os.getenv("REDIS_URL", "redis://localhost:6379/0") + +# ── Execution mode ──────────────────────────────────────────────────────────── +EXECUTION_MODE: str = os.getenv("EXECUTION_MODE", "paper").lower() +LIVE_MODE_ENABLED: bool = EXECUTION_MODE == "live" + +# ── Risk defaults ───────────────────────────────────────────────────────────── +MAX_DAILY_LOSS_PCT: float = float(os.getenv("MAX_DAILY_LOSS_PCT", "2.0")) +MAX_POSITION_SIZE_PCT: float = float(os.getenv("MAX_POSITION_SIZE_PCT", "5.0")) +MAX_SPREAD_MULTIPLE: float = float(os.getenv("MAX_SPREAD_MULTIPLE", "3.0")) +MIN_MODEL_CONFIDENCE: float = float(os.getenv("MIN_MODEL_CONFIDENCE", "0.55")) +MIN_DATA_QUALITY_SCORE: float = float(os.getenv("MIN_DATA_QUALITY_SCORE", "0.80")) + +# ── Order book processing ───────────────────────────────────────────────────── +OB_DEPTH_LEVELS: int = 10 # number of price levels to track each side +OB_WINDOW_SECONDS: int = 90 # rolling window for imbalance metrics +SWEEP_PRICE_LEVELS: int = 3 # consecutive levels crossed = sweep event +ABSORPTION_MIN_PRINTS: int = 5 # minimum prints to qualify as absorption event +SPOOF_MIN_CANCEL_RATIO: float = 4.0 # cancel-to-add ratio threshold for spoof flag + +# ── Feature engineering ─────────────────────────────────────────────────────── +FEATURE_WINDOWS: list[int] = [5, 10, 30, 60, 300] # seconds +RSI_PERIOD: int = 14 +ATR_PERIOD: int = 14 +VWAP_SESSION_RESET: bool = True + +# ── Compliance hardcoded constants ──────────────────────────────────────────── +IDENTITY_CLAIM_BLOCKED_TERMS: list[str] = [ + "blackrock", "citadel", "vanguard", "jane street", "two sigma", + "renaissance", "bridgewater", "millennium", "point72", "d.e. shaw", + "virtu", "jump trading", "susquehanna", "optiver", "imc", +] +COMPLIANCE_LIMITATIONS_STATEMENT: str = ( + "Level 2 data does not reveal the actual identity of market participants. " + "All behavioral classifications are probabilistic estimates based on " + "observable, legally accessible market data only." +) diff --git a/src/dashboard/__init__.py b/src/dashboard/__init__.py new file mode 100644 index 0000000..7b680e8 --- /dev/null +++ b/src/dashboard/__init__.py @@ -0,0 +1,3 @@ +""" +dashboard/__init__.py +""" diff --git a/src/dashboard/app.py b/src/dashboard/app.py new file mode 100644 index 0000000..c7c3318 --- /dev/null +++ b/src/dashboard/app.py @@ -0,0 +1,304 @@ +""" +dashboard/app.py – Streamlit Research Dashboard (Phase 1). + +Provides a live visual interface for: + - Order book depth chart (bid / ask ladders) + - Bid/ask imbalance gauge + - Order flow imbalance time series + - Behavioral signal panel + - Decision Parliament disposition + - Chat interface + - Audit log viewer + +Run with: + streamlit run src/dashboard/app.py +""" + +from __future__ import annotations + +import asyncio +import time +from collections import deque +from threading import Thread + +import plotly.graph_objects as go +import streamlit as st + +from src.config import DEFAULT_SYMBOL +from src.data_intake.sample_feed import SampleFeedAdapter +from src.orchestrator import Orchestrator + +# ── Page config ─────────────────────────────────────────────────────────────── +st.set_page_config( + page_title="Microstructure Intelligence", + page_icon="📊", + layout="wide", + initial_sidebar_state="expanded", +) + +# ── Sidebar ─────────────────────────────────────────────────────────────────── +with st.sidebar: + st.title("⚙️ Configuration") + symbol = st.text_input("Symbol", value=DEFAULT_SYMBOL).upper() + feed_source = st.selectbox("Data Feed", ["Sample (offline)", "Polygon.io", "Alpaca"]) + st.caption("Live feeds require API keys in .env") + st.divider() + st.markdown( + "**Execution Mode**\n" + "🟡 PAPER ONLY \n" + "Live execution is disabled by default. \n" + "Human approval required for live trading." + ) + st.divider() + st.caption( + "⚠️ All analysis is for research purposes only. " + "Behavioural classifications are probabilistic estimates. " + "Participant identity cannot be inferred from Level 2 data." + ) + +# ── Session state ───────────────────────────────────────────────────────────── +if "orchestrator" not in st.session_state or st.session_state.get("symbol") != symbol: + st.session_state.orchestrator = Orchestrator(symbol) + st.session_state.symbol = symbol + st.session_state.obi_history = deque(maxlen=120) + st.session_state.ofi_history = deque(maxlen=120) + st.session_state.mid_history = deque(maxlen=120) + st.session_state.ts_history = deque(maxlen=120) + st.session_state.chat_history = [] + st.session_state.latest_result = None + +orch: Orchestrator = st.session_state.orchestrator + +# ── Header ──────────────────────────────────────────────────────────────────── +st.title(f"📊 Institutional Microstructure Intelligence — {symbol}") +st.caption("Phase 1 Research Prototype | Paper-Only | Compliance-Aware | Audit-Driven") + +# ── Tick one snapshot synchronously (for demo / refresh) ───────────────────── +feed = SampleFeedAdapter(symbol=symbol, base_price=185.0) +snapshot = feed.get_snapshot() +trades = feed._generate_trades() + +result = orch.process(snapshot, trades) +st.session_state.latest_result = result + +state = orch.latest_state +features = orch.latest_features +health = orch.latest_health + +# Store history +ts = time.time() +st.session_state.obi_history.append(state.book_imbalance) +st.session_state.ofi_history.append(state.order_flow_imbalance) +st.session_state.mid_history.append(state.mid_price) +st.session_state.ts_history.append(ts) + +# ── Top KPI row ─────────────────────────────────────────────────────────────── +col1, col2, col3, col4, col5, col6 = st.columns(6) + +with col1: + st.metric("Mid Price", f"${state.mid_price:.2f}") +with col2: + st.metric("Spread", f"${state.spread:.4f}", f"{state.spread_multiple:.1f}×avg") +with col3: + obi = state.book_imbalance + col3.metric("Book Imbalance", f"{obi:+.3f}", "Bullish" if obi > 0.15 else "Bearish" if obi < -0.15 else "Neutral") +with col4: + st.metric("Absorption", f"{state.absorption_score:.2f}") +with col5: + st.metric("Sweep", f"{state.sweep_intensity:.2f}") +with col6: + disp_color = { + "paper_trade_approved": "🟢", + "watchlist": "🟡", + "research_approved": "🔵", + "risk_veto": "🔴", + "data_insufficient": "⚠️", + "blocked": "⛔", + "human_review": "🟠", + "rejected": "⚫", + }.get(result.disposition, "⚪") + st.metric("Disposition", f"{disp_color} {result.disposition.replace('_', ' ').upper()}") + +st.divider() + +# ── Two-column layout: order book + charts ──────────────────────────────────── +left, right = st.columns([1, 2]) + +with left: + st.subheader("📖 Order Book Depth") + if state.bids and state.asks: + bid_prices = [lvl.price for lvl in state.bids[:8]] + bid_sizes = [lvl.size for lvl in state.bids[:8]] + ask_prices = [lvl.price for lvl in state.asks[:8]] + ask_sizes = [lvl.size for lvl in state.asks[:8]] + + fig_ob = go.Figure() + fig_ob.add_trace(go.Bar( + x=bid_sizes, + y=[f"${p:.2f}" for p in bid_prices], + orientation="h", + name="Bid", + marker_color="rgba(0, 200, 100, 0.7)", + )) + fig_ob.add_trace(go.Bar( + x=ask_sizes, + y=[f"${p:.2f}" for p in ask_prices], + orientation="h", + name="Ask", + marker_color="rgba(220, 60, 60, 0.7)", + )) + fig_ob.update_layout( + barmode="overlay", + height=350, + margin=dict(l=10, r=10, t=10, b=10), + legend=dict(x=0.7, y=1), + xaxis_title="Size", + ) + st.plotly_chart(fig_ob, use_container_width=True) + else: + st.info("Waiting for order book data…") + + # Signal panel + st.subheader("🔔 Active Signals") + for sig in state.signals: + if "stacking" in sig.lower() or "absorption" in sig.lower(): + st.warning(sig) + elif "sweep" in sig.lower() or "spoof" in sig.lower(): + st.error(sig) + elif "no significant" in sig.lower(): + st.info(sig) + else: + st.success(sig) + +with right: + st.subheader("📈 Order Book Imbalance (rolling)") + obi_list = list(st.session_state.obi_history) + ofi_list = list(st.session_state.ofi_history) + mid_list = list(st.session_state.mid_history) + + if obi_list: + fig_obi = go.Figure() + fig_obi.add_trace(go.Scatter( + y=obi_list, mode="lines", name="OBI", + line=dict(color="royalblue", width=2), + )) + fig_obi.add_trace(go.Scatter( + y=ofi_list, mode="lines", name="OFI", + line=dict(color="darkorange", width=2), + )) + fig_obi.add_hline(y=0, line_dash="dash", line_color="gray") + fig_obi.add_hline(y=0.3, line_dash="dot", line_color="green", annotation_text="Bullish threshold") + fig_obi.add_hline(y=-0.3, line_dash="dot", line_color="red", annotation_text="Bearish threshold") + fig_obi.update_layout( + height=240, + margin=dict(l=10, r=10, t=10, b=10), + yaxis=dict(range=[-1.1, 1.1]), + ) + st.plotly_chart(fig_obi, use_container_width=True) + + st.subheader("💹 Mid Price") + if mid_list: + fig_mid = go.Figure() + fig_mid.add_trace(go.Scatter( + y=mid_list, mode="lines", name="Mid", + line=dict(color="white", width=2), + fill="tozeroy", fillcolor="rgba(100,100,200,0.15)", + )) + if features and features.vwap: + fig_mid.add_hline(y=features.vwap, line_dash="dash", line_color="gold", + annotation_text="VWAP") + fig_mid.update_layout( + height=200, + margin=dict(l=10, r=10, t=10, b=10), + ) + st.plotly_chart(fig_mid, use_container_width=True) + +# ── Behavioral / Institutional panel ───────────────────────────────────────── +st.divider() +st.subheader("🔍 Behavioral Intelligence") +b1, b2, b3, b4 = st.columns(4) + +with b1: + st.metric("Accumulation Score", f"{features.accumulation_score:.2f}" if features else "—") +with b2: + st.metric("Distribution Score", f"{features.distribution_score:.2f}" if features else "—") +with b3: + st.metric("Inst. Footprint Prob", f"{features.institutional_footprint_prob:.0%}" if features else "—") +with b4: + st.metric("Liquidity Trap Risk", f"{features.liquidity_trap_risk:.2f}" if features else "—") + +# ── Technical indicators ───────────────────────────────────────────────────── +st.divider() +st.subheader("📐 Technical Indicators") +t1, t2, t3, t4, t5 = st.columns(5) +with t1: + st.metric("RSI (14)", f"{features.rsi_14:.1f}" if features and not __import__('math').isnan(features.rsi_14) else "—") +with t2: + st.metric("ATR (14)", f"{features.atr_14:.4f}" if features and not __import__('math').isnan(features.atr_14) else "—") +with t3: + st.metric("MACD", f"{features.macd_line:.4f}" if features and not __import__('math').isnan(features.macd_line) else "—") +with t4: + st.metric("BB Width", f"{features.bb_width:.4f}" if features and not __import__('math').isnan(features.bb_width) else "—") +with t5: + st.metric("RVOL", f"{features.rvol:.2f}×" if features else "—") + +# ── Data quality ────────────────────────────────────────────────────────────── +st.divider() +st.subheader("🛡️ Data & Risk Status") +dq1, dq2, dq3, dq4, dq5 = st.columns(5) +with dq1: + q = health.quality_score if health else 0 + st.metric("Feed Quality", f"{q:.0%}", delta="OK" if q >= 0.8 else "LOW") +with dq2: + st.metric("Latency", f"{health.latency_ms:.0f}ms" if health else "—") +with dq3: + st.metric("Risk Status", result.risk_status.upper()) +with dq4: + st.metric("Compliance", result.compliance_status.upper()) +with dq5: + st.metric("Spoof-like Score", f"{state.spoof_like_score:.2f}") + +# ── Decision Parliament summary ─────────────────────────────────────────────── +st.divider() +st.subheader("🏛️ Decision Parliament") +with st.expander("View Parliament Votes", expanded=True): + parl_cols = st.columns(5) + parl_cols[0].metric("OB Analyst", f"{result.ob_analyst_vote.upper()}", f"{result.ob_analyst_confidence:.0%}") + parl_cols[1].metric("Market DNA", result.dna_regime.upper(), f"{result.dna_confidence:.0%}") + parl_cols[2].metric("Footprint", result.inst_footprint_label.replace("_", " ").upper(), f"{result.inst_footprint_prob:.0%}") + parl_cols[3].metric("Risk Gov.", result.risk_status.upper()) + parl_cols[4].metric("Compliance", result.compliance_status.upper()) + st.info(f"**Reasoning:** {result.reasoning}") + st.caption(f"⚠️ {result.limitations}") + +# ── Chatbot interface ───────────────────────────────────────────────────────── +st.divider() +st.subheader("💬 Research Chatbot") + +chat_container = st.container() +with chat_container: + for msg in st.session_state.chat_history[-10:]: + role = msg["role"] + content = msg["content"] + if role == "user": + st.markdown(f"**You:** {content}") + else: + st.markdown(f"**System:**\n```\n{content}\n```") + +user_input = st.chat_input("Ask about the order book… (e.g. 'analyze AAPL', 'explain the risk', 'help')") +if user_input: + st.session_state.chat_history.append({"role": "user", "content": user_input}) + response = orch.chat(user_input) + st.session_state.chat_history.append({"role": "assistant", "content": response.text}) + st.rerun() + +# ── Auto-refresh ────────────────────────────────────────────────────────────── +st.divider() +refresh = st.button("🔄 Refresh Analysis") +if refresh: + st.rerun() + +st.caption( + "Auto-refresh: press Refresh or use `streamlit run` with `--server.runOnSave true`. " + "Live feed streaming will be added in Phase 3." +) diff --git a/src/data_intake/__init__.py b/src/data_intake/__init__.py new file mode 100644 index 0000000..3f7251d --- /dev/null +++ b/src/data_intake/__init__.py @@ -0,0 +1,3 @@ +""" +data_intake/__init__.py +""" diff --git a/src/data_intake/data_integrity_agent.py b/src/data_intake/data_integrity_agent.py new file mode 100644 index 0000000..7a30c08 --- /dev/null +++ b/src/data_intake/data_integrity_agent.py @@ -0,0 +1,127 @@ +""" +data_intake/data_integrity_agent.py – Data Integrity Agent (Phase 1). + +Monitors incoming snapshots and trade streams for: + - Stale quotes (no update within threshold) + - Bad ticks (price outliers) + - Missing data (gaps in sequence numbers) + - Feed latency + - Zero-size quotes + +Produces a FeedHealthReport that downstream agents consume before processing. +A failing health check can halt or flag downstream processing. +""" + +from __future__ import annotations + +import statistics +import time +from collections import deque + +from src.data_intake.models import FeedHealthReport, OrderBookSnapshot, Trade + + +class DataIntegrityAgent: + """ + Stateful feed monitor. + + Call check() after each (snapshot, trades) pair. The returned + FeedHealthReport is passed to the audit ledger and the agentic + orchestration layer. + """ + + STALE_THRESHOLD_MS = 5_000 # quote older than 5s → stale + PRICE_OUTLIER_SIGMA = 5.0 # prints > 5 σ from rolling mean → bad tick + PRICE_HISTORY_SIZE = 200 # rolling window for outlier detection + MAX_LATENCY_MS = 2_000 # >2 s feed lag → degraded health + + def __init__(self) -> None: + self._last_ts_ns: int | None = None + self._prev_sequence: int | None = None + self._price_history: deque[float] = deque(maxlen=self.PRICE_HISTORY_SIZE) + self._latency_samples: deque[float] = deque(maxlen=50) + + # ── Public ──────────────────────────────────────────────────────────────── + + def check( + self, + snapshot: OrderBookSnapshot, + trades: list[Trade], + ) -> FeedHealthReport: + now_ns = time.time_ns() + notes: list[str] = [] + bad_ticks = 0 + gap_detected = False + + # ── Staleness ──────────────────────────────────────────────────────── + stale = False + if self._last_ts_ns is not None: + age_ms = (now_ns - snapshot.timestamp_ns) / 1e6 + if age_ms > self.STALE_THRESHOLD_MS: + stale = True + notes.append(f"Stale quote: {age_ms:.0f}ms old") + self._last_ts_ns = snapshot.timestamp_ns + + # ── Sequence gap ───────────────────────────────────────────────────── + if snapshot.sequence is not None: + if self._prev_sequence is not None and snapshot.sequence != self._prev_sequence + 1: + gap_detected = True + notes.append( + f"Sequence gap: expected {self._prev_sequence + 1}, got {snapshot.sequence}" + ) + self._prev_sequence = snapshot.sequence + + # ── Latency estimate ───────────────────────────────────────────────── + latency_ms = (now_ns - snapshot.timestamp_ns) / 1e6 + self._latency_samples.append(latency_ms) + avg_latency = statistics.mean(self._latency_samples) + if avg_latency > self.MAX_LATENCY_MS: + notes.append(f"High feed latency: avg {avg_latency:.0f}ms") + + # ── Bad tick detection ──────────────────────────────────────────────── + for trade in trades: + self._price_history.append(trade.price) + if len(self._price_history) >= 30: + mu = statistics.mean(self._price_history) + sigma = statistics.pstdev(self._price_history) + if sigma > 0 and abs(trade.price - mu) > self.PRICE_OUTLIER_SIGMA * sigma: + bad_ticks += 1 + notes.append( + f"Bad tick: price {trade.price} is {abs(trade.price - mu)/sigma:.1f}σ from mean" + ) + + # ── Zero-size quote check ───────────────────────────────────────────── + missing_ticks = 0 + if snapshot.best_bid and snapshot.best_bid.size == 0: + missing_ticks += 1 + notes.append("Zero-size best bid") + if snapshot.best_ask and snapshot.best_ask.size == 0: + missing_ticks += 1 + notes.append("Zero-size best ask") + + # ── Quality score ───────────────────────────────────────────────────── + quality = 1.0 + if stale: + quality -= 0.30 + if gap_detected: + quality -= 0.20 + if bad_ticks > 0: + quality -= min(0.20, bad_ticks * 0.05) + if avg_latency > self.MAX_LATENCY_MS: + quality -= 0.15 + if missing_ticks > 0: + quality -= 0.10 + quality = max(0.0, round(quality, 3)) + + return FeedHealthReport( + feed=snapshot.feed, + symbol=snapshot.symbol, + quality_score=quality, + latency_ms=round(avg_latency, 2), + missing_ticks=missing_ticks, + stale_quote=stale, + bad_ticks=bad_ticks, + gap_detected=gap_detected, + timestamp_ns=now_ns, + notes=notes, + ) diff --git a/src/data_intake/models.py b/src/data_intake/models.py new file mode 100644 index 0000000..dbd8616 --- /dev/null +++ b/src/data_intake/models.py @@ -0,0 +1,131 @@ +""" +data_intake/models.py – Canonical data models for all incoming market data. + +Every feed adapter normalises raw exchange/broker data into these structures +before it reaches the order book engine. This decouples feed-specific formats +from all downstream processing. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class Quote: + """Level 1 NBBO quote snapshot.""" + symbol: str + bid_price: float + bid_size: float + ask_price: float + ask_size: float + timestamp_ns: int = field(default_factory=lambda: time.time_ns()) + feed: str = "unknown" + + @property + def spread(self) -> float: + return round(self.ask_price - self.bid_price, 6) + + @property + def mid_price(self) -> float: + return round((self.bid_price + self.ask_price) / 2, 6) + + @property + def microprice(self) -> float: + """Weighted mid-price – pulls toward the thinner side of the book.""" + total = self.bid_size + self.ask_size + if total == 0: + return self.mid_price + return round( + (self.bid_price * self.ask_size + self.ask_price * self.bid_size) / total, 6 + ) + + +@dataclass +class L2Level: + """A single price level on one side of the order book.""" + price: float + size: float + + +@dataclass +class OrderBookSnapshot: + """Full Level 2 order book snapshot at a point in time.""" + symbol: str + bids: list[L2Level] # sorted descending by price + asks: list[L2Level] # sorted ascending by price + timestamp_ns: int = field(default_factory=lambda: time.time_ns()) + feed: str = "unknown" + sequence: Optional[int] = None + + @property + def best_bid(self) -> Optional[L2Level]: + return self.bids[0] if self.bids else None + + @property + def best_ask(self) -> Optional[L2Level]: + return self.asks[0] if self.asks else None + + @property + def spread(self) -> float: + if self.best_bid and self.best_ask: + return round(self.best_ask.price - self.best_bid.price, 6) + return float("nan") + + @property + def mid_price(self) -> float: + if self.best_bid and self.best_ask: + return round((self.best_bid.price + self.best_ask.price) / 2, 6) + return float("nan") + + def total_bid_depth(self, levels: int = 10) -> float: + return sum(lvl.size for lvl in self.bids[:levels]) + + def total_ask_depth(self, levels: int = 10) -> float: + return sum(lvl.size for lvl in self.asks[:levels]) + + +@dataclass +class Trade: + """A single time-and-sales print.""" + symbol: str + price: float + size: float + side: str # "buy" | "sell" | "unknown" + timestamp_ns: int = field(default_factory=lambda: time.time_ns()) + feed: str = "unknown" + conditions: list[str] = field(default_factory=list) + + @property + def is_buy_aggressor(self) -> bool: + return self.side == "buy" + + @property + def notional(self) -> float: + return round(self.price * self.size, 2) + + +@dataclass +class FeedHealthReport: + """Data-quality report produced by the Data Integrity Agent.""" + feed: str + symbol: str + quality_score: float # 0.0 – 1.0 + latency_ms: float + missing_ticks: int + stale_quote: bool + bad_ticks: int + gap_detected: bool + timestamp_ns: int = field(default_factory=lambda: time.time_ns()) + notes: list[str] = field(default_factory=list) + + @property + def is_healthy(self) -> bool: + return ( + self.quality_score >= 0.80 + and not self.stale_quote + and not self.gap_detected + and self.latency_ms < 2000 + ) diff --git a/src/data_intake/polygon_feed.py b/src/data_intake/polygon_feed.py new file mode 100644 index 0000000..3b754e1 --- /dev/null +++ b/src/data_intake/polygon_feed.py @@ -0,0 +1,132 @@ +""" +data_intake/polygon_feed.py – Polygon.io Level 2 WebSocket feed adapter. + +Connects to Polygon's WebSocket stream for real-time Level 2 (NBBO + quotes) +and time-and-sales data. Normalises all messages into canonical data models. + +Requires: + POLYGON_API_KEY set in environment / .env + +Documentation: https://polygon.io/docs/stocks/ws_stocks_q +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +from typing import AsyncIterator + +import websockets + +from src.config import POLYGON_API_KEY +from src.data_intake.models import L2Level, OrderBookSnapshot, Quote, Trade + +logger = logging.getLogger(__name__) + +POLYGON_WS_URL = "wss://delayed.polygon.io/stocks" # use wss://socket.polygon.io for live + + +class PolygonFeedAdapter: + """ + Polygon.io WebSocket feed adapter. + + Subscribes to: + Q.* – NBBO quote updates (best bid/ask) + T.* – Trade prints (time and sales) + + Note: Full Level 2 depth-of-book requires Polygon's 'Starter' plan or above. + This adapter uses NBBO quotes as a Level 1 / pseudo-Level 2 baseline and can + be extended with Polygon's full order book feed when the subscription allows. + """ + + def __init__(self, symbol: str, api_key: str = POLYGON_API_KEY) -> None: + if not api_key: + raise ValueError( + "POLYGON_API_KEY is not set. Add it to your .env file. " + "See .env.example for guidance." + ) + self.symbol = symbol.upper() + self.api_key = api_key + self._latest_snapshot: OrderBookSnapshot | None = None + self._pending_trades: list[Trade] = [] + + async def stream(self) -> AsyncIterator[tuple[OrderBookSnapshot, list[Trade]]]: + """Yield (snapshot, trades) pairs as they arrive from Polygon.""" + async with websockets.connect(POLYGON_WS_URL) as ws: + await self._authenticate(ws) + await self._subscribe(ws) + logger.info("Polygon feed active for %s", self.symbol) + + async for raw in ws: + messages = json.loads(raw) + for msg in messages: + ev = msg.get("ev") + if ev == "Q": + self._handle_quote(msg) + elif ev == "T": + self._handle_trade(msg) + + if self._latest_snapshot is not None: + trades, self._pending_trades = self._pending_trades, [] + yield self._latest_snapshot, trades + + # ── Private ─────────────────────────────────────────────────────────────── + + async def _authenticate(self, ws) -> None: + await ws.send(json.dumps({"action": "auth", "params": self.api_key})) + resp = json.loads(await ws.recv()) + if any(m.get("status") == "auth_success" for m in resp): + logger.info("Polygon authentication successful") + else: + raise ConnectionError(f"Polygon auth failed: {resp}") + + async def _subscribe(self, ws) -> None: + channels = f"Q.{self.symbol},T.{self.symbol}" + await ws.send(json.dumps({"action": "subscribe", "params": channels})) + + def _handle_quote(self, msg: dict) -> None: + bid_p = float(msg.get("bp", 0)) + ask_p = float(msg.get("ap", 0)) + bid_s = float(msg.get("bs", 0)) + ask_s = float(msg.get("as", 0)) + ts = msg.get("t", time.time_ns() // 1_000_000) * 1_000_000 # ms → ns + + # Polygon NBBO gives us one level; build a minimal pseudo-L2 structure. + # Full L2 depth requires the Nasdaq TotalView or Polygon depth feed. + self._latest_snapshot = OrderBookSnapshot( + symbol=self.symbol, + bids=[L2Level(price=bid_p, size=bid_s)], + asks=[L2Level(price=ask_p, size=ask_s)], + timestamp_ns=ts, + feed="polygon", + ) + + def _handle_trade(self, msg: dict) -> None: + price = float(msg.get("p", 0)) + size = float(msg.get("s", 0)) + conditions = msg.get("c", []) + ts = msg.get("t", time.time_ns() // 1_000_000) * 1_000_000 + + # Polygon does not always classify aggressor side; infer from quote. + side = "unknown" + if self._latest_snapshot: + bb = self._latest_snapshot.best_bid + ba = self._latest_snapshot.best_ask + if ba and abs(price - ba.price) < 0.005: + side = "buy" + elif bb and abs(price - bb.price) < 0.005: + side = "sell" + + self._pending_trades.append( + Trade( + symbol=self.symbol, + price=price, + size=size, + side=side, + timestamp_ns=ts, + feed="polygon", + conditions=[str(c) for c in conditions], + ) + ) diff --git a/src/data_intake/sample_feed.py b/src/data_intake/sample_feed.py new file mode 100644 index 0000000..2184e28 --- /dev/null +++ b/src/data_intake/sample_feed.py @@ -0,0 +1,154 @@ +""" +data_intake/sample_feed.py – Synthetic / replay feed for development and testing. + +Generates realistic-looking Level 2 order book snapshots and trade prints +without requiring live API credentials. All downstream code is feed-agnostic +and works identically with real feeds. + +Usage: + feed = SampleFeedAdapter(symbol="NVDA", base_price=500.0) + async for snapshot, trades in feed.stream(): + process(snapshot, trades) +""" + +from __future__ import annotations + +import asyncio +import random +import time +from typing import AsyncIterator + +from src.data_intake.models import L2Level, OrderBookSnapshot, Quote, Trade + + +class SampleFeedAdapter: + """ + Deterministic-ish synthetic market data generator. + + Simulates a random-walk mid-price with realistic spread and depth dynamics, + including occasional absorption, sweep, stacking, and pulling events. + """ + + def __init__( + self, + symbol: str = "AAPL", + base_price: float = 185.0, + tick_size: float = 0.01, + depth_levels: int = 10, + interval_ms: int = 250, + seed: int | None = None, + ) -> None: + self.symbol = symbol + self.tick_size = tick_size + self.depth_levels = depth_levels + self.interval_ms = interval_ms + self._price = base_price + self._rng = random.Random(seed) + self._sequence = 0 + + # ── Public interface ────────────────────────────────────────────────────── + + async def stream( + self, max_ticks: int | None = None + ) -> AsyncIterator[tuple[OrderBookSnapshot, list[Trade]]]: + """Yield (OrderBookSnapshot, list[Trade]) at configured interval.""" + tick = 0 + while max_ticks is None or tick < max_ticks: + self._evolve_price() + snapshot = self._build_snapshot() + trades = self._generate_trades() + yield snapshot, trades + tick += 1 + await asyncio.sleep(self.interval_ms / 1000.0) + + def get_snapshot(self) -> OrderBookSnapshot: + """Synchronous single snapshot, useful for testing.""" + self._evolve_price() + return self._build_snapshot() + + def get_quote(self) -> Quote: + snap = self._build_snapshot() + bb = snap.best_bid + ba = snap.best_ask + return Quote( + symbol=self.symbol, + bid_price=bb.price if bb else self._price - self.tick_size, + bid_size=bb.size if bb else 100, + ask_price=ba.price if ba else self._price + self.tick_size, + ask_size=ba.size if ba else 100, + feed="sample", + ) + + # ── Private helpers ─────────────────────────────────────────────────────── + + def _evolve_price(self) -> None: + """Random walk with occasional regime shifts.""" + drift = self._rng.gauss(0, 0.03) + # Occasional momentum burst or mean-reversion + if self._rng.random() < 0.05: + drift += self._rng.choice([-0.15, 0.15]) + self._price = round( + max(self._price + drift, self.tick_size), 2 + ) + + def _build_snapshot(self) -> OrderBookSnapshot: + self._sequence += 1 + spread_ticks = self._rng.randint(1, 4) + half_spread = (spread_ticks * self.tick_size) / 2 + best_bid = round(self._price - half_spread, 2) + best_ask = round(self._price + half_spread, 2) + + bids: list[L2Level] = [] + asks: list[L2Level] = [] + + for i in range(self.depth_levels): + bid_price = round(best_bid - i * self.tick_size * self._rng.uniform(1, 3), 2) + ask_price = round(best_ask + i * self.tick_size * self._rng.uniform(1, 3), 2) + + # Occasional stacking (large size at a level) + bid_size = self._rng.randint(100, 800) + ask_size = self._rng.randint(100, 800) + if i == 0 and self._rng.random() < 0.1: + bid_size = self._rng.randint(2000, 8000) # stacking event + if i == 1 and self._rng.random() < 0.08: + ask_size = self._rng.randint(2000, 8000) # ask wall + + bids.append(L2Level(price=bid_price, size=bid_size)) + asks.append(L2Level(price=ask_price, size=ask_size)) + + return OrderBookSnapshot( + symbol=self.symbol, + bids=bids, + asks=asks, + timestamp_ns=time.time_ns(), + feed="sample", + sequence=self._sequence, + ) + + def _generate_trades(self) -> list[Trade]: + """Produce 0–5 trade prints per tick.""" + trades: list[Trade] = [] + n = self._rng.choices([0, 1, 2, 3, 4, 5], weights=[30, 35, 20, 8, 5, 2])[0] + + for _ in range(n): + side = self._rng.choice(["buy", "sell"]) + price_offset = self._rng.uniform(0, 0.03) + price = round( + self._price + (price_offset if side == "buy" else -price_offset), 2 + ) + size = self._rng.randint(10, 500) + # Occasional large institutional-size print + if self._rng.random() < 0.03: + size = self._rng.randint(1000, 10000) + + trades.append( + Trade( + symbol=self.symbol, + price=price, + size=size, + side=side, + timestamp_ns=time.time_ns(), + feed="sample", + ) + ) + return trades diff --git a/src/features/__init__.py b/src/features/__init__.py new file mode 100644 index 0000000..ca2a261 --- /dev/null +++ b/src/features/__init__.py @@ -0,0 +1,3 @@ +""" +features/__init__.py +""" diff --git a/src/features/engineer.py b/src/features/engineer.py new file mode 100644 index 0000000..49b9a20 --- /dev/null +++ b/src/features/engineer.py @@ -0,0 +1,444 @@ +""" +features/engineer.py – Feature Engineering Layer (Phase 1). + +Produces a named, versioned feature vector from each OrderBookState. +This vector is the primary input to Phase 2 machine learning models. + +Feature groups: + 1. Core order book features + 2. Technical indicator features (VWAP, RVOL, RSI, ATR, MACD, BB) + 3. Behavioral composite scores + +All features are returned as a FeatureVector dataclass and also as a +flat dict suitable for serialisation (pandas, Parquet, model inference). +""" + +from __future__ import annotations + +import math +import time +from collections import deque +from dataclasses import dataclass, field, asdict +from typing import Optional + +from src.config import FEATURE_WINDOWS, RSI_PERIOD, ATR_PERIOD +from src.data_intake.models import Trade +from src.order_book.engine import OrderBookState + + +FEATURE_VERSION = "1.0" + + +@dataclass +class FeatureVector: + """ + Complete ML-ready feature vector produced at each tick. + All values are float. NaN indicates unavailable / insufficient history. + """ + # ── Metadata ────────────────────────────────────────────────────────────── + feature_version: str + symbol: str + timestamp_ns: int + + # ── Core order book features ────────────────────────────────────────────── + bid_ask_imbalance: float + depth_weighted_imbalance: float + spread: float + mid_price: float + microprice: float + microprice_bias: float # microprice - mid_price (signed pressure) + order_flow_imbalance: float + bid_depth_change_pct: float + ask_depth_change_pct: float + absorption_score: float + sweep_intensity: float + spoof_like_score: float + iceberg_like_score: float + bid_replenishment_rate: float + ask_replenishment_rate: float + volume_at_bid: float + volume_at_ask: float + rolling_buy_sell_ratio: float + spread_multiple: float + stacking_bid: float # bool encoded as 0/1 + stacking_ask: float + pulling_bid: float + pulling_ask: float + + # ── Multi-window OBI deltas ─────────────────────────────────────────────── + obi_5s: float + obi_10s: float + obi_30s: float + ofi_5s: float + ofi_30s: float + + # ── Technical indicators ────────────────────────────────────────────────── + vwap: float + vwap_deviation: float # (mid - vwap) / vwap + rvol: float # relative volume vs 20-bar average + rsi_14: float + atr_14: float + macd_line: float + macd_signal: float + macd_histogram: float + bb_upper: float + bb_lower: float + bb_width: float + bb_pct_b: float # %B: position within Bollinger Bands + obv: float # on-balance volume + ma_slope_5: float # slope of 5-bar MA + ma_slope_20: float + + # ── Behavioral composite scores ─────────────────────────────────────────── + accumulation_score: float # composite: OBI + absorption + replenishment + VWAP support + distribution_score: float # composite: negative OBI + ask stacking + fading + institutional_footprint_prob: float + momentum_ignition_risk: float + liquidity_trap_risk: float + breakout_confirmation_score: float + false_breakout_prob: float # placeholder until ML model available + + def to_dict(self) -> dict: + return asdict(self) + + def to_flat_dict(self) -> dict[str, float | str | int]: + """Flatten for pandas / Parquet storage.""" + return self.to_dict() + + +class FeatureEngineer: + """ + Stateful feature engineer. + + Maintains rolling price/volume/indicator history per symbol. + Call update() after each (OrderBookState, list[Trade]) pair. + """ + + def __init__(self, symbol: str) -> None: + self.symbol = symbol + self._mid_prices: deque[float] = deque(maxlen=200) + self._volumes: deque[float] = deque(maxlen=200) + self._highs: deque[float] = deque(maxlen=200) + self._lows: deque[float] = deque(maxlen=200) + self._closes: deque[float] = deque(maxlen=200) + self._obi_ts: deque[tuple[float, int]] = deque(maxlen=500) # (obi, ts_ns) + self._ofi_ts: deque[tuple[float, int]] = deque(maxlen=500) + self._vwap_num: float = 0.0 # cumulative price × volume + self._vwap_den: float = 0.0 # cumulative volume + self._obv: float = 0.0 + self._prev_mid: Optional[float] = None + self._avg_vol_20: deque[float] = deque(maxlen=20) + + # ── Public ──────────────────────────────────────────────────────────────── + + def update( + self, + state: OrderBookState, + trades: list[Trade], + ) -> FeatureVector: + """Ingest new state and trades; return updated FeatureVector.""" + mp = state.mid_price + trade_vol = sum(t.size for t in trades) + trade_value = sum(t.price * t.size for t in trades) + + # Update OHLCV-like rolling series + self._mid_prices.append(mp) + self._closes.append(mp) + self._highs.append(max((t.price for t in trades), default=mp)) + self._lows.append(min((t.price for t in trades), default=mp)) + self._volumes.append(trade_vol) + self._avg_vol_20.append(trade_vol) + + # VWAP + self._vwap_num += trade_value + self._vwap_den += trade_vol + vwap = (self._vwap_num / self._vwap_den) if self._vwap_den > 0 else mp + vwap_dev = ((mp - vwap) / vwap) if vwap > 0 else 0.0 + + # OBV + if self._prev_mid is not None: + if mp > self._prev_mid: + self._obv += trade_vol + elif mp < self._prev_mid: + self._obv -= trade_vol + self._prev_mid = mp + + # Relative volume + avg_vol = sum(self._avg_vol_20) / len(self._avg_vol_20) if self._avg_vol_20 else 1 + rvol = trade_vol / avg_vol if avg_vol > 0 else 1.0 + + # Technical indicators + rsi = self._rsi(list(self._closes), RSI_PERIOD) + atr = self._atr( + list(self._highs), list(self._lows), list(self._closes), ATR_PERIOD + ) + macd_l, macd_s, macd_h = self._macd(list(self._closes)) + bb_u, bb_l, bb_w, bb_pctb = self._bollinger(list(self._closes)) + ma5 = self._ma_slope(list(self._closes), 5) + ma20 = self._ma_slope(list(self._closes), 20) + + # Windowed OBI / OFI + self._obi_ts.append((state.book_imbalance, state.timestamp_ns)) + self._ofi_ts.append((state.order_flow_imbalance, state.timestamp_ns)) + obi_5, obi_10, obi_30 = self._windowed_mean(self._obi_ts, [5, 10, 30]) + ofi_5, ofi_30 = self._windowed_mean(self._ofi_ts, [5, 30])[:2] + + # Behavioral composite scores + accum = self._accumulation_score(state, vwap_dev) + distrib = self._distribution_score(state, vwap_dev) + inst_prob = self._institutional_footprint_prob(state, accum, distrib) + mig_risk = self._momentum_ignition_risk(state, rvol) + lt_risk = self._liquidity_trap_risk(state, atr) + bo_score = self._breakout_confirmation_score(state, rvol) + fb_prob = self._false_breakout_prob(state) # heuristic placeholder + + return FeatureVector( + feature_version=FEATURE_VERSION, + symbol=self.symbol, + timestamp_ns=state.timestamp_ns, + # Core OB + bid_ask_imbalance=state.book_imbalance, + depth_weighted_imbalance=state.depth_weighted_imbalance, + spread=state.spread, + mid_price=state.mid_price, + microprice=state.microprice, + microprice_bias=round(state.microprice - state.mid_price, 6), + order_flow_imbalance=state.order_flow_imbalance, + bid_depth_change_pct=state.bid_depth_change_pct, + ask_depth_change_pct=state.ask_depth_change_pct, + absorption_score=state.absorption_score, + sweep_intensity=state.sweep_intensity, + spoof_like_score=state.spoof_like_score, + iceberg_like_score=state.iceberg_like_score, + bid_replenishment_rate=state.bid_replenishment_rate, + ask_replenishment_rate=state.ask_replenishment_rate, + volume_at_bid=state.volume_at_bid, + volume_at_ask=state.volume_at_ask, + rolling_buy_sell_ratio=state.rolling_buy_sell_ratio, + spread_multiple=state.spread_multiple, + stacking_bid=float(state.stacking_bid), + stacking_ask=float(state.stacking_ask), + pulling_bid=float(state.pulling_bid), + pulling_ask=float(state.pulling_ask), + # Multi-window + obi_5s=obi_5, + obi_10s=obi_10, + obi_30s=obi_30, + ofi_5s=ofi_5, + ofi_30s=ofi_30, + # Technical + vwap=round(vwap, 4), + vwap_deviation=round(vwap_dev, 6), + rvol=round(rvol, 4), + rsi_14=rsi, + atr_14=atr, + macd_line=macd_l, + macd_signal=macd_s, + macd_histogram=macd_h, + bb_upper=bb_u, + bb_lower=bb_l, + bb_width=bb_w, + bb_pct_b=bb_pctb, + obv=round(self._obv, 2), + ma_slope_5=ma5, + ma_slope_20=ma20, + # Behavioral + accumulation_score=accum, + distribution_score=distrib, + institutional_footprint_prob=inst_prob, + momentum_ignition_risk=mig_risk, + liquidity_trap_risk=lt_risk, + breakout_confirmation_score=bo_score, + false_breakout_prob=fb_prob, + ) + + # ── Indicator implementations ───────────────────────────────────────────── + + @staticmethod + def _rsi(closes: list[float], period: int) -> float: + if len(closes) < period + 1: + return float("nan") + gains, losses = [], [] + for i in range(1, len(closes)): + chg = closes[i] - closes[i - 1] + gains.append(max(chg, 0)) + losses.append(max(-chg, 0)) + avg_gain = sum(gains[-period:]) / period + avg_loss = sum(losses[-period:]) / period + if avg_loss == 0: + return 100.0 + rs = avg_gain / avg_loss + return round(100 - (100 / (1 + rs)), 4) + + @staticmethod + def _atr( + highs: list[float], lows: list[float], closes: list[float], period: int + ) -> float: + if len(closes) < period + 1: + return float("nan") + trs = [] + for i in range(1, len(closes)): + tr = max( + highs[i] - lows[i], + abs(highs[i] - closes[i - 1]), + abs(lows[i] - closes[i - 1]), + ) + trs.append(tr) + return round(sum(trs[-period:]) / period, 6) + + @staticmethod + def _ema(data: list[float], period: int) -> list[float]: + if not data: + return [] + k = 2 / (period + 1) + ema = [data[0]] + for v in data[1:]: + ema.append(v * k + ema[-1] * (1 - k)) + return ema + + def _macd( + self, closes: list[float], fast: int = 12, slow: int = 26, signal: int = 9 + ) -> tuple[float, float, float]: + if len(closes) < slow + signal: + return float("nan"), float("nan"), float("nan") + ema_fast = self._ema(closes, fast) + ema_slow = self._ema(closes, slow) + macd_line = [f - s for f, s in zip(ema_fast, ema_slow)] + macd_signal = self._ema(macd_line, signal) + if not macd_signal: + return float("nan"), float("nan"), float("nan") + ml = round(macd_line[-1], 6) + ms = round(macd_signal[-1], 6) + return ml, ms, round(ml - ms, 6) + + @staticmethod + def _bollinger(closes: list[float], period: int = 20, std_dev: float = 2.0): + if len(closes) < period: + return float("nan"), float("nan"), float("nan"), float("nan") + window = closes[-period:] + mean = sum(window) / period + variance = sum((x - mean) ** 2 for x in window) / period + std = math.sqrt(variance) + upper = round(mean + std_dev * std, 4) + lower = round(mean - std_dev * std, 4) + width = round(upper - lower, 6) + last = closes[-1] + pct_b = round((last - lower) / width, 4) if width > 0 else 0.5 + return upper, lower, width, pct_b + + @staticmethod + def _ma_slope(closes: list[float], period: int) -> float: + if len(closes) < period + 1: + return float("nan") + ma_now = sum(closes[-period:]) / period + ma_prev = sum(closes[-(period + 1):-1]) / period + return round(ma_now - ma_prev, 6) + + @staticmethod + def _windowed_mean( + buffer: deque[tuple[float, int]], windows_s: list[int] + ) -> list[float]: + now = time.time_ns() + results = [] + for w in windows_s: + cutoff = now - w * 1_000_000_000 + vals = [v for v, ts in buffer if ts >= cutoff] + results.append(round(sum(vals) / len(vals), 4) if vals else float("nan")) + return results + + # ── Behavioral composite scores ─────────────────────────────────────────── + + @staticmethod + def _accumulation_score(state: OrderBookState, vwap_dev: float) -> float: + score = 0.0 + if state.book_imbalance > 0.2: + score += 0.25 + if state.absorption_score > 0.5: + score += 0.25 + if state.bid_replenishment_rate > 0.3: + score += 0.20 + if vwap_dev > -0.002: # price near or above VWAP + score += 0.15 + if state.order_flow_imbalance > 0.1: + score += 0.15 + return round(min(score, 1.0), 3) + + @staticmethod + def _distribution_score(state: OrderBookState, vwap_dev: float) -> float: + score = 0.0 + if state.book_imbalance < -0.2: + score += 0.25 + if state.stacking_ask: + score += 0.25 + if vwap_dev < 0.002: # price near or below VWAP + score += 0.15 + if state.order_flow_imbalance < -0.1: + score += 0.20 + if state.ask_replenishment_rate > 0.3: + score += 0.15 + return round(min(score, 1.0), 3) + + @staticmethod + def _institutional_footprint_prob( + state: OrderBookState, accum: float, distrib: float + ) -> float: + """ + Combined probability that observed activity resembles institutional-style + participation (either accumulation or distribution style). + + All outputs from this score MUST be accompanied by the compliance limitations + statement in any user-facing response. + """ + base = max(accum, distrib) + if state.iceberg_like_score > 0.4: + base += 0.10 + if state.absorption_score > 0.6: + base += 0.08 + return round(min(base, 1.0), 3) + + @staticmethod + def _momentum_ignition_risk(state: OrderBookState, rvol: float) -> float: + score = 0.0 + if state.sweep_intensity > 0.5: + score += 0.4 + if rvol > 2.0: + score += 0.3 + if state.spoof_like_score > 0.3: + score += 0.3 + return round(min(score, 1.0), 3) + + @staticmethod + def _liquidity_trap_risk(state: OrderBookState, atr: float) -> float: + score = 0.0 + if state.spread_multiple > 2.0: + score += 0.4 + if not math.isnan(atr) and atr > 0 and state.spread > atr * 0.1: + score += 0.3 + if state.pulling_bid or state.pulling_ask: + score += 0.3 + return round(min(score, 1.0), 3) + + @staticmethod + def _breakout_confirmation_score(state: OrderBookState, rvol: float) -> float: + score = 0.0 + if state.sweep_intensity > 0.4: + score += 0.3 + if rvol > 1.5: + score += 0.3 + if state.book_imbalance > 0.3: + score += 0.2 + if state.absorption_score < 0.3: # low absorption = price can move + score += 0.2 + return round(min(score, 1.0), 3) + + @staticmethod + def _false_breakout_prob(state: OrderBookState) -> float: + """Heuristic placeholder – Phase 2 will replace with trained classifier.""" + score = 0.0 + if state.spoof_like_score > 0.3: + score += 0.4 + if state.pulling_bid or state.pulling_ask: + score += 0.3 + if state.spread_multiple > 2.0: + score += 0.3 + return round(min(score, 1.0), 3) diff --git a/src/orchestrator.py b/src/orchestrator.py new file mode 100644 index 0000000..de9b31f --- /dev/null +++ b/src/orchestrator.py @@ -0,0 +1,166 @@ +""" +orchestrator.py – Agentic Orchestration Layer (Phase 1). + +Wires together all agents into a single synchronous processing pipeline. +In Phase 3, this will be replaced by a LangGraph / CrewAI graph. + +Pipeline per tick: + 1. Data Integrity Agent → FeedHealthReport + 2. Order Book Engine → OrderBookState + 3. Feature Engineer → FeatureVector + 4. Order Book Analyst → OrderBookAnalystReport + 5. Market-DNA Detector → MarketDNAReport + 6. Institutional Footprint Agent → InstitutionalFootprintReport + 7. Risk Governor → RiskGovernorReport + 8. Compliance Agent → ComplianceReport + 9. Decision Parliament → DecisionParliamentResult + 10. Audit Ledger → writes JSONL record + 11. Chatbot Interface → human-readable response (on demand) +""" + +from __future__ import annotations + +from src.agents.compliance_agent import ComplianceAgent +from src.agents.decision_parliament import DecisionParliament +from src.agents.institutional_footprint import InstitutionalFootprintAgent +from src.agents.market_dna_detector import MarketDNADetectorAgent +from src.agents.models import DecisionParliamentResult +from src.agents.order_book_analyst import OrderBookAnalystAgent +from src.agents.risk_governor import RiskGovernorAgent +from src.audit.ledger import AuditLedger +from src.chatbot.interface import ChatbotInterface, ChatbotResponse +from src.data_intake.data_integrity_agent import DataIntegrityAgent +from src.data_intake.models import FeedHealthReport, OrderBookSnapshot, Trade +from src.features.engineer import FeatureEngineer, FeatureVector +from src.order_book.engine import OrderBookEngine, OrderBookState + + +class Orchestrator: + """ + Central pipeline coordinator. + + Instantiate once per symbol and call process() after each + (snapshot, trades) pair from the feed. + """ + + def __init__(self, symbol: str, feature_log_every: int = 10) -> None: + self.symbol = symbol + self._feature_log_every = feature_log_every + self._tick = 0 + + # ── Agents ──────────────────────────────────────────────────────────── + self._integrity = DataIntegrityAgent() + self._ob_engine = OrderBookEngine() + self._feature_eng = FeatureEngineer(symbol) + self._ob_analyst = OrderBookAnalystAgent() + self._dna_detector = MarketDNADetectorAgent() + self._inst_footprint = InstitutionalFootprintAgent() + self._risk_governor = RiskGovernorAgent() + self._compliance = ComplianceAgent() + self._parliament = DecisionParliament() + self._chatbot = ChatbotInterface() + self._ledger = AuditLedger() + + # ── Latest state cache (for dashboard reads) ────────────────────────── + self.latest_health: FeedHealthReport | None = None + self.latest_state: OrderBookState | None = None + self.latest_features: FeatureVector | None = None + self.latest_result: DecisionParliamentResult | None = None + + # ── Public ──────────────────────────────────────────────────────────────── + + def process( + self, + snapshot: OrderBookSnapshot, + trades: list[Trade], + ) -> DecisionParliamentResult: + """Run full pipeline for one tick. Returns the parliament result.""" + self._tick += 1 + + # 1. Data integrity + health = self._integrity.check(snapshot, trades) + self._ledger.log_health(health) + + # 2. Order book processing + state = self._ob_engine.process(snapshot, trades) + + # 3. Feature engineering + features = self._feature_eng.update(state, trades) + + # 4. Order book analyst + ob_report = self._ob_analyst.analyse(state, features) + + # 5. Market-DNA detector + dna_report = self._dna_detector.classify(state, features) + + # 6. Institutional footprint + inst_report = self._inst_footprint.analyse(state, features) + + # 7. Risk governor + risk_report = self._risk_governor.evaluate(state, features, health) + + # 8. Compliance (check the human explanation before delivery) + pre_text = ( + f"{ob_report.directional_bias} {dna_report.regime} " + f"{inst_report.behavioral_label} {' '.join(state.signals)}" + ) + compliance_report = self._compliance.check_output(self.symbol, pre_text) + + # 9. Decision parliament + result = self._parliament.deliberate( + ob_report, dna_report, inst_report, risk_report, compliance_report + ) + + # 10. Audit + self._ledger.log_decision(result) + if self._tick % self._feature_log_every == 0: + self._ledger.log_features(features) + + # Cache + self.latest_health = health + self.latest_state = state + self.latest_features = features + self.latest_result = result + + return result + + def chat(self, user_input: str) -> ChatbotResponse: + """Answer a user query using the latest pipeline state.""" + if not self.latest_result: + return self._chatbot.respond(user_input) + + # Re-run compliance on the chatbot output + state = self.latest_state + features = self.latest_features + result = self.latest_result + + # Reconstruct minimal agent reports from cached state + ob_report = self._ob_analyst.analyse(state, features) + dna_report = self._dna_detector.classify(state, features) + inst_report = self._inst_footprint.analyse(state, features) + risk_report = self._risk_governor.evaluate( + state, features, self.latest_health + ) + + response = self._chatbot.respond( + user_input, + ob_report=ob_report, + dna_report=dna_report, + inst_report=inst_report, + risk_report=risk_report, + parliament_result=result, + ) + + # Final compliance gate on the response text + compliance_check = self._compliance.check_output(self.symbol, response.text) + if compliance_check.status == "blocked": + response.compliance_status = "blocked" + response.text = ( + "⛔ Response blocked by Compliance Agent. " + + (compliance_check.violations[0] if compliance_check.violations else "") + ) + + return response + + def close(self) -> None: + self._ledger.close() diff --git a/src/order_book/__init__.py b/src/order_book/__init__.py new file mode 100644 index 0000000..53d2bc4 --- /dev/null +++ b/src/order_book/__init__.py @@ -0,0 +1,3 @@ +""" +order_book/__init__.py +""" diff --git a/src/order_book/engine.py b/src/order_book/engine.py new file mode 100644 index 0000000..07cf1e2 --- /dev/null +++ b/src/order_book/engine.py @@ -0,0 +1,516 @@ +""" +order_book/engine.py – Level 2 Order Book Processing Engine (Phase 1). + +Transforms raw order book snapshots and trade prints into structured +behavioral intelligence signals. + +Tracks and computes: + - Bid / ask depth per level + - Order book imbalance (OBI) + - Depth-weighted imbalance + - Spread (current, rolling average, multiple) + - Liquidity stacking events + - Liquidity pulling events + - Spoof-like behavior signal + - Iceberg-like behavior signal + - Absorption score + - Sweep events + - Bid / ask replenishment + - Queue pressure + - Volume at bid vs ask + - Microprice + +All computed values feed directly into the Feature Engineering Layer. +""" + +from __future__ import annotations + +import time +from collections import deque +from dataclasses import dataclass, field +from typing import Optional + +from src.config import ( + ABSORPTION_MIN_PRINTS, + OB_DEPTH_LEVELS, + OB_WINDOW_SECONDS, + SPOOF_MIN_CANCEL_RATIO, + SWEEP_PRICE_LEVELS, +) +from src.data_intake.models import L2Level, OrderBookSnapshot, Trade + + +@dataclass +class OrderBookState: + """ + Complete processed state produced after each snapshot update. + All downstream agents read from this structure. + """ + symbol: str + timestamp_ns: int + + # ── Raw depth ───────────────────────────────────────────────────────────── + bids: list[L2Level] + asks: list[L2Level] + best_bid: Optional[L2Level] + best_ask: Optional[L2Level] + spread: float + mid_price: float + microprice: float + + # ── Imbalance metrics ───────────────────────────────────────────────────── + book_imbalance: float # (bid_vol - ask_vol) / (bid_vol + ask_vol) [-1, 1] + depth_weighted_imbalance: float # weighted version across levels + order_flow_imbalance: float # cumulative signed volume, rolling window + + # ── Liquidity events ───────────────────────────────────────────────────── + bid_depth_change_pct: float # % change in total bid depth vs previous snapshot + ask_depth_change_pct: float + stacking_bid: bool # abnormally large size appeared at bid + stacking_ask: bool + pulling_bid: bool # large size disappeared from bid without a fill + pulling_ask: bool + + # ── Behavioral scores (0.0 – 1.0) ──────────────────────────────────────── + absorption_score: float # large prints absorbed without price continuation + sweep_intensity: float # consecutive level crossings + spoof_like_score: float # cancel-rate heuristic near best bid/ask + iceberg_like_score: float # repeated replenishment after partial fills + bid_replenishment_rate: float # bid side recovery rate + ask_replenishment_rate: float # ask side recovery rate + + # ── Volume analysis ─────────────────────────────────────────────────────── + volume_at_bid: float # inferred sell-aggressor volume, rolling window + volume_at_ask: float # inferred buy-aggressor volume, rolling window + rolling_buy_sell_ratio: float # buy_vol / (buy_vol + sell_vol) + + # ── Spread context ──────────────────────────────────────────────────────── + spread_avg_20: float # 20-snapshot rolling average spread + spread_multiple: float # spread / spread_avg_20 + + # ── Human-readable signal summary ──────────────────────────────────────── + signals: list[str] = field(default_factory=list) + + +class OrderBookEngine: + """ + Stateful processor that converts each (OrderBookSnapshot, list[Trade]) + pair into a fully computed OrderBookState. + + Maintains rolling buffers to compute windowed metrics. + """ + + _STACK_THRESHOLD_MULTIPLE = 4.0 # level size > 4× mean depth → stacking + _PULL_THRESHOLD_MULTIPLE = 3.0 # size drop > 3× mean depth without fill → pulling + _REPLENISH_THRESHOLD = 0.5 # depth recovered ≥50% after being hit + + def __init__(self, depth_levels: int = OB_DEPTH_LEVELS) -> None: + self.depth_levels = depth_levels + self._prev_snapshot: Optional[OrderBookSnapshot] = None + + # Rolling buffers (keyed by symbol; single-symbol engine for Phase 1) + window = OB_WINDOW_SECONDS * 4 # ~250ms ticks + self._spreads: deque[float] = deque(maxlen=20) + self._ofi_buffer: deque[tuple[float, int]] = deque(maxlen=window) # (signed_vol, ts_ns) + self._trade_buffer: deque[Trade] = deque(maxlen=window) + self._bid_depth_history: deque[float] = deque(maxlen=window) + self._ask_depth_history: deque[float] = deque(maxlen=window) + self._absorption_prints: deque[tuple[float, float]] = deque(maxlen=50) # (price, size) + self._replenish_bid_hits: deque[float] = deque(maxlen=30) + self._replenish_ask_hits: deque[float] = deque(maxlen=30) + self._cancel_add_ratio_buffer: deque[float] = deque(maxlen=20) + + # ── Public interface ────────────────────────────────────────────────────── + + def process( + self, + snapshot: OrderBookSnapshot, + trades: list[Trade], + ) -> OrderBookState: + """Compute and return a fully populated OrderBookState.""" + bids = snapshot.bids[: self.depth_levels] + asks = snapshot.asks[: self.depth_levels] + + bid_vol = sum(lvl.size for lvl in bids) + ask_vol = sum(lvl.size for lvl in asks) + + # Ingest trades into rolling buffers + for t in trades: + self._trade_buffer.append(t) + signed = t.size if t.side == "buy" else -t.size if t.side == "sell" else 0 + self._ofi_buffer.append((signed, t.timestamp_ns)) + + # Rolling buffers + self._bid_depth_history.append(bid_vol) + self._ask_depth_history.append(ask_vol) + spread = snapshot.spread + if not _is_nan(spread): + self._spreads.append(spread) + + # Compute all signals + book_imbalance = self._book_imbalance(bid_vol, ask_vol) + dwi = self._depth_weighted_imbalance(bids, asks) + ofi = self._order_flow_imbalance() + bid_chg, ask_chg = self._depth_change_pct() + stacking_bid, stacking_ask = self._detect_stacking(bids, asks) + pulling_bid, pulling_ask = self._detect_pulling(bids, asks) + absorption = self._absorption_score(trades, snapshot) + sweep = self._sweep_intensity(trades, snapshot) + spoof = self._spoof_like_score(snapshot) + iceberg = self._iceberg_like_score(trades) + bid_replen, ask_replen = self._replenishment_rates(snapshot, trades) + vol_bid, vol_ask, bsr = self._volume_analysis() + spread_avg = sum(self._spreads) / len(self._spreads) if self._spreads else spread + spread_mult = (spread / spread_avg) if spread_avg > 0 and not _is_nan(spread) else 1.0 + + mp = snapshot.mid_price + uprice = self._microprice(snapshot) + + signals = self._build_signals( + book_imbalance=book_imbalance, + bid_chg=bid_chg, + ask_chg=ask_chg, + stacking_bid=stacking_bid, + stacking_ask=stacking_ask, + pulling_bid=pulling_bid, + pulling_ask=pulling_ask, + absorption=absorption, + sweep=sweep, + spoof=spoof, + iceberg=iceberg, + spread_mult=spread_mult, + bsr=bsr, + ) + + self._prev_snapshot = snapshot + + return OrderBookState( + symbol=snapshot.symbol, + timestamp_ns=snapshot.timestamp_ns, + bids=bids, + asks=asks, + best_bid=snapshot.best_bid, + best_ask=snapshot.best_ask, + spread=spread, + mid_price=mp, + microprice=uprice, + book_imbalance=book_imbalance, + depth_weighted_imbalance=dwi, + order_flow_imbalance=ofi, + bid_depth_change_pct=bid_chg, + ask_depth_change_pct=ask_chg, + stacking_bid=stacking_bid, + stacking_ask=stacking_ask, + pulling_bid=pulling_bid, + pulling_ask=pulling_ask, + absorption_score=absorption, + sweep_intensity=sweep, + spoof_like_score=spoof, + iceberg_like_score=iceberg, + bid_replenishment_rate=bid_replen, + ask_replenishment_rate=ask_replen, + volume_at_bid=vol_bid, + volume_at_ask=vol_ask, + rolling_buy_sell_ratio=bsr, + spread_avg_20=round(spread_avg, 6), + spread_multiple=round(spread_mult, 3), + signals=signals, + ) + + # ── Signal computation ──────────────────────────────────────────────────── + + @staticmethod + def _book_imbalance(bid_vol: float, ask_vol: float) -> float: + total = bid_vol + ask_vol + return round((bid_vol - ask_vol) / total, 4) if total > 0 else 0.0 + + @staticmethod + def _depth_weighted_imbalance( + bids: list[L2Level], asks: list[L2Level] + ) -> float: + """Imbalance weighted by inverse price distance from best bid/ask.""" + if not bids or not asks: + return 0.0 + best_b = bids[0].price + best_a = asks[0].price + bid_w = sum(lvl.size / (1 + abs(lvl.price - best_b)) for lvl in bids) + ask_w = sum(lvl.size / (1 + abs(lvl.price - best_a)) for lvl in asks) + total = bid_w + ask_w + return round((bid_w - ask_w) / total, 4) if total > 0 else 0.0 + + def _order_flow_imbalance(self) -> float: + """Cumulative signed volume over rolling window, normalised.""" + if not self._ofi_buffer: + return 0.0 + cutoff_ns = time.time_ns() - OB_WINDOW_SECONDS * 1_000_000_000 + vals = [sv for sv, ts in self._ofi_buffer if ts >= cutoff_ns] + if not vals: + return 0.0 + total_abs = sum(abs(v) for v in vals) + return round(sum(vals) / total_abs, 4) if total_abs > 0 else 0.0 + + def _depth_change_pct(self) -> tuple[float, float]: + if len(self._bid_depth_history) < 2: + return 0.0, 0.0 + prev_b = self._bid_depth_history[-2] + prev_a = self._ask_depth_history[-2] + curr_b = self._bid_depth_history[-1] + curr_a = self._ask_depth_history[-1] + bid_chg = ((curr_b - prev_b) / prev_b * 100) if prev_b > 0 else 0.0 + ask_chg = ((curr_a - prev_a) / prev_a * 100) if prev_a > 0 else 0.0 + return round(bid_chg, 2), round(ask_chg, 2) + + def _detect_stacking( + self, bids: list[L2Level], asks: list[L2Level] + ) -> tuple[bool, bool]: + """A stacking event is flagged when a single level carries ≥4× mean depth.""" + if len(bids) < 2 or len(asks) < 2: + return False, False + bid_mean = sum(lvl.size for lvl in bids) / len(bids) + ask_mean = sum(lvl.size for lvl in asks) / len(asks) + stacking_b = bids[0].size >= self._STACK_THRESHOLD_MULTIPLE * bid_mean + stacking_a = asks[0].size >= self._STACK_THRESHOLD_MULTIPLE * ask_mean + return stacking_b, stacking_a + + def _detect_pulling( + self, bids: list[L2Level], asks: list[L2Level] + ) -> tuple[bool, bool]: + """Pulling detected when best-level depth drops sharply without a matching trade.""" + pulling_b = pulling_a = False + if self._prev_snapshot: + prev_bb = self._prev_snapshot.best_bid + prev_ba = self._prev_snapshot.best_ask + curr_bb = bids[0] if bids else None + curr_ba = asks[0] if asks else None + + recent_trade_vol = sum( + t.size for t in list(self._trade_buffer)[-5:] if t.side == "sell" + ) + if ( + prev_bb + and curr_bb + and prev_bb.price == curr_bb.price + and prev_bb.size > 0 + ): + drop = (prev_bb.size - curr_bb.size) / prev_bb.size + if drop >= 0.5 and recent_trade_vol < prev_bb.size * 0.1: + pulling_b = True + + recent_buy_vol = sum( + t.size for t in list(self._trade_buffer)[-5:] if t.side == "buy" + ) + if ( + prev_ba + and curr_ba + and prev_ba.price == curr_ba.price + and prev_ba.size > 0 + ): + drop = (prev_ba.size - curr_ba.size) / prev_ba.size + if drop >= 0.5 and recent_buy_vol < prev_ba.size * 0.1: + pulling_a = True + + return pulling_b, pulling_a + + def _absorption_score( + self, trades: list[Trade], snapshot: OrderBookSnapshot + ) -> float: + """ + Absorption: large aggressive prints execute but price does not move + in the aggressor's direction. + + Score = fraction of recent large prints that were 'absorbed'. + """ + large_threshold = 500 # shares – configurable + absorbed = 0 + total_large = 0 + prev = self._prev_snapshot + + for t in trades: + if t.size < large_threshold: + continue + total_large += 1 + if prev: + if t.side == "buy" and snapshot.mid_price <= prev.mid_price + 0.02: + absorbed += 1 + elif t.side == "sell" and snapshot.mid_price >= prev.mid_price - 0.02: + absorbed += 1 + + if total_large < ABSORPTION_MIN_PRINTS: + # Accumulate across recent history + recent = [t for t in self._trade_buffer if t.size >= large_threshold] + total_large = len(recent) + if total_large == 0: + return 0.0 + absorbed = sum( + 1 for t in recent + if snapshot.mid_price is not None + ) # simplified for Phase 1 + + return round(min(absorbed / max(total_large, 1), 1.0), 3) + + def _sweep_intensity( + self, trades: list[Trade], snapshot: OrderBookSnapshot + ) -> float: + """ + Sweep: consecutive aggressive prints crossing multiple price levels. + Returns a normalised intensity in [0, 1]. + """ + if len(trades) < 2: + return 0.0 + price_levels_hit = len({round(t.price, 2) for t in trades}) + return round(min(price_levels_hit / SWEEP_PRICE_LEVELS, 1.0), 3) + + def _spoof_like_score(self, snapshot: OrderBookSnapshot) -> float: + """ + Heuristic: high cancel-to-add ratio near best bid/ask suggests spoof-like activity. + Phase 1 approximation: compare current best-level depth to rolling mean. + Elevated cancel-to-add will be tracked via tick-by-tick diffs in Phase 2+. + """ + if not self._bid_depth_history or len(self._bid_depth_history) < 5: + return 0.0 + bb = snapshot.best_bid + if bb is None: + return 0.0 + mean_bid = sum(list(self._bid_depth_history)[-10:]) / min( + len(self._bid_depth_history), 10 + ) + # If best bid is dramatically larger than recent mean, flag potential stacking + # that could be pulled (spoof-like pattern precursor) + if mean_bid > 0 and bb.size / mean_bid >= SPOOF_MIN_CANCEL_RATIO: + return round(min((bb.size / mean_bid - SPOOF_MIN_CANCEL_RATIO) / 4, 1.0), 3) + return 0.0 + + def _iceberg_like_score(self, trades: list[Trade]) -> float: + """ + Iceberg heuristic: repeated fills of similar size at the same price level, + with the level apparently refreshing after each fill. + """ + if len(trades) < 3: + return 0.0 + sizes = [t.size for t in trades if t.size > 100] + if len(sizes) < 3: + return 0.0 + # Check for clustering of similar print sizes + mean_s = sum(sizes) / len(sizes) + similar = sum(1 for s in sizes if abs(s - mean_s) / mean_s < 0.15) + return round(similar / len(sizes), 3) + + def _replenishment_rates( + self, snapshot: OrderBookSnapshot, trades: list[Trade] + ) -> tuple[float, float]: + """Track bid/ask replenishment after being hit.""" + prev = self._prev_snapshot + if not prev: + return 0.0, 0.0 + + bid_replen = 0.0 + ask_replen = 0.0 + + prev_bb = prev.best_bid + curr_bb = snapshot.best_bid + if prev_bb and curr_bb and prev_bb.price == curr_bb.price: + sell_vol = sum(t.size for t in trades if t.side == "sell") + if sell_vol > 0 and curr_bb.size > prev_bb.size * 0.5: + bid_replen = min(curr_bb.size / (prev_bb.size + sell_vol), 1.0) + + prev_ba = prev.best_ask + curr_ba = snapshot.best_ask + if prev_ba and curr_ba and prev_ba.price == curr_ba.price: + buy_vol = sum(t.size for t in trades if t.side == "buy") + if buy_vol > 0 and curr_ba.size > prev_ba.size * 0.5: + ask_replen = min(curr_ba.size / (prev_ba.size + buy_vol), 1.0) + + return round(bid_replen, 3), round(ask_replen, 3) + + def _volume_analysis(self) -> tuple[float, float, float]: + """Buy vs sell volume over rolling window.""" + cutoff_ns = time.time_ns() - OB_WINDOW_SECONDS * 1_000_000_000 + recent = [t for t in self._trade_buffer if t.timestamp_ns >= cutoff_ns] + vol_buy = sum(t.size for t in recent if t.side == "buy") + vol_sell = sum(t.size for t in recent if t.side == "sell") + total = vol_buy + vol_sell + bsr = vol_buy / total if total > 0 else 0.5 + return vol_sell, vol_buy, round(bsr, 4) + + @staticmethod + def _microprice(snapshot: OrderBookSnapshot) -> float: + bb = snapshot.best_bid + ba = snapshot.best_ask + if not bb or not ba: + return snapshot.mid_price + total = bb.size + ba.size + if total == 0: + return snapshot.mid_price + return round((bb.price * ba.size + ba.price * bb.size) / total, 6) + + def _build_signals(self, **kwargs) -> list[str]: + """Generate human-readable signal strings for the chatbot layer.""" + signals: list[str] = [] + obi = kwargs["book_imbalance"] + bid_chg = kwargs["bid_chg"] + ask_chg = kwargs["ask_chg"] + spread_mult = kwargs["spread_mult"] + + if obi > 0.3: + signals.append(f"Order book imbalance favours buyers (+{obi:.2f}).") + elif obi < -0.3: + signals.append(f"Order book imbalance favours sellers ({obi:.2f}).") + + if kwargs["stacking_bid"]: + signals.append("Bid-side stacking detected: abnormally large size at best bid.") + if kwargs["stacking_ask"]: + signals.append("Ask-side stacking detected: abnormally large size at best ask.") + + if kwargs["pulling_bid"]: + signals.append("Bid-side pulling detected: large size removed without execution.") + if kwargs["pulling_ask"]: + signals.append("Ask-side pulling detected: large size removed without execution.") + + if bid_chg > 20: + signals.append(f"Bid-side liquidity increased {bid_chg:.1f}% since last tick.") + elif bid_chg < -20: + signals.append(f"Bid-side liquidity decreased {abs(bid_chg):.1f}% since last tick.") + + if ask_chg > 20: + signals.append(f"Ask-side liquidity increased {ask_chg:.1f}% since last tick.") + elif ask_chg < -20: + signals.append(f"Ask-side liquidity decreased {abs(ask_chg):.1f}% since last tick.") + + if kwargs["absorption"] > 0.6: + signals.append( + f"Absorption detected: large prints absorbed without price continuation " + f"(score {kwargs['absorption']:.2f})." + ) + + if kwargs["sweep"] > 0.6: + signals.append(f"Sweep activity detected (intensity {kwargs['sweep']:.2f}).") + + if kwargs["spoof"] > 0.3: + signals.append( + f"Spoof-like signal: elevated cancel-to-add pattern detected near best bid/ask " + f"(score {kwargs['spoof']:.2f}). Treat with caution." + ) + + if kwargs["iceberg"] > 0.5: + signals.append( + f"Iceberg-like activity: repeated similar-size fills detected " + f"(score {kwargs['iceberg']:.2f})." + ) + + if spread_mult > 2.5: + signals.append( + f"Spread elevated: {spread_mult:.1f}× the 20-snapshot average. Execution risk increased." + ) + + if not signals: + signals.append("No significant order book signals detected at this time.") + + return signals + + +def _is_nan(v: float) -> bool: + try: + import math + return math.isnan(v) + except Exception: + return False diff --git a/src/utils/__init__.py b/src/utils/__init__.py new file mode 100644 index 0000000..1c80b56 --- /dev/null +++ b/src/utils/__init__.py @@ -0,0 +1,3 @@ +""" +utils/__init__.py +""" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..4fee185 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +""" +tests/__init__.py +""" diff --git a/tests/test_agents.py b/tests/test_agents.py new file mode 100644 index 0000000..784553c --- /dev/null +++ b/tests/test_agents.py @@ -0,0 +1,255 @@ +""" +tests/test_agents.py – Unit tests for all agent modules. +""" +import pytest +from src.data_intake.models import L2Level, OrderBookSnapshot, Trade, FeedHealthReport +from src.order_book.engine import OrderBookEngine +from src.features.engineer import FeatureEngineer +from src.agents.order_book_analyst import OrderBookAnalystAgent +from src.agents.market_dna_detector import MarketDNADetectorAgent +from src.agents.institutional_footprint import InstitutionalFootprintAgent +from src.agents.risk_governor import RiskGovernorAgent +from src.agents.compliance_agent import ComplianceAgent +from src.agents.decision_parliament import DecisionParliament + + +def _make_pipeline(bid_size=500, ask_size=400, mid=100.0, spread_mult=1.0): + engine = OrderBookEngine(depth_levels=5) + bids = [L2Level(price=round(mid - 0.05 * spread_mult - i * 0.01, 2), size=bid_size) for i in range(5)] + asks = [L2Level(price=round(mid + 0.05 * spread_mult + i * 0.01, 2), size=ask_size) for i in range(5)] + snap = OrderBookSnapshot(symbol="TEST", bids=bids, asks=asks, feed="test") + trades = [Trade("TEST", price=mid, size=100, side="buy")] + state = engine.process(snap, trades) + eng = FeatureEngineer("TEST") + features = eng.update(state, trades) + health = FeedHealthReport( + feed="test", symbol="TEST", quality_score=0.95, latency_ms=10, + missing_ticks=0, stale_quote=False, bad_ticks=0, gap_detected=False, + ) + return state, features, health + + +class TestOrderBookAnalystAgent: + def test_returns_report(self): + state, features, _ = _make_pipeline() + agent = OrderBookAnalystAgent() + report = agent.analyse(state, features) + assert report.symbol == "TEST" + assert report.directional_bias in {"bullish", "bearish", "neutral"} + assert 0 <= report.bias_confidence <= 1 + + def test_bullish_when_bid_dominates(self): + state, features, _ = _make_pipeline(bid_size=5000, ask_size=200) + agent = OrderBookAnalystAgent() + report = agent.analyse(state, features) + assert report.directional_bias in {"bullish", "neutral"} + + def test_bearish_when_ask_dominates(self): + state, features, _ = _make_pipeline(bid_size=200, ask_size=5000) + agent = OrderBookAnalystAgent() + report = agent.analyse(state, features) + assert report.directional_bias in {"bearish", "neutral"} + + +class TestMarketDNADetector: + def test_returns_report(self): + state, features, _ = _make_pipeline() + agent = MarketDNADetectorAgent() + report = agent.classify(state, features) + assert report.regime is not None + assert 0 <= report.regime_confidence <= 1 + assert isinstance(report.supporting_signals, list) + + def test_valid_regime_label(self): + state, features, _ = _make_pipeline() + agent = MarketDNADetectorAgent() + report = agent.classify(state, features) + valid = { + "trending_up", "trending_down", "ranging", "breakout", "reversal", + "compression", "expansion", "accumulation", "distribution", "trap", "ambiguous" + } + assert report.regime in valid + + +class TestInstitutionalFootprintAgent: + def test_returns_report_with_limitations(self): + state, features, _ = _make_pipeline() + agent = InstitutionalFootprintAgent() + report = agent.analyse(state, features) + assert report.limitations != "" + assert "level 2 data" in report.limitations.lower() + + def test_probability_in_range(self): + state, features, _ = _make_pipeline() + agent = InstitutionalFootprintAgent() + report = agent.analyse(state, features) + assert 0 <= report.probability <= 1 + + def test_no_identity_claims_in_label(self): + """Label must never contain a named institution.""" + state, features, _ = _make_pipeline() + agent = InstitutionalFootprintAgent() + report = agent.analyse(state, features) + from src.config import IDENTITY_CLAIM_BLOCKED_TERMS + label_lower = report.behavioral_label.lower() + for term in IDENTITY_CLAIM_BLOCKED_TERMS: + assert term not in label_lower, f"Identity claim found in label: {term}" + + +class TestRiskGovernorAgent: + def test_clear_status_normal_conditions(self): + state, features, health = _make_pipeline() + agent = RiskGovernorAgent() + report = agent.evaluate(state, features, health) + assert report.risk_status in {"clear", "caution", "veto"} + assert 0 <= report.risk_score <= 1 + + def test_veto_on_bad_data_quality(self): + state, features, _ = _make_pipeline() + bad_health = FeedHealthReport( + feed="test", symbol="TEST", quality_score=0.30, + latency_ms=10, missing_ticks=0, stale_quote=True, + bad_ticks=0, gap_detected=True, + ) + agent = RiskGovernorAgent() + report = agent.evaluate(state, features, bad_health) + assert report.veto is True + assert report.risk_status == "veto" + + def test_caution_or_veto_on_wide_spread(self): + """Wide spread should elevate risk status.""" + state, features, health = _make_pipeline(spread_mult=5.0) + # Artificially set spread_multiple + state.spread_multiple = 5.0 + agent = RiskGovernorAgent() + report = agent.evaluate(state, features, health) + assert report.risk_status in {"caution", "veto"} + + +class TestComplianceAgent: + def test_approves_clean_text(self): + agent = ComplianceAgent() + result = agent.check_output("TEST", "Order book imbalance is positive.") + assert result.status == "approved" + + def test_blocks_named_institution(self): + agent = ComplianceAgent() + result = agent.check_output("TEST", "BlackRock is buying at this level.") + assert result.status == "blocked" + assert len(result.violations) > 0 + + def test_blocks_identity_claim_phrase(self): + agent = ComplianceAgent() + result = agent.check_output("TEST", "Citadel is buying here.") + assert result.status == "blocked" + + def test_flags_certainty_language(self): + agent = ComplianceAgent() + result = agent.check_output("TEST", "This is definitely bullish behavior.") + assert result.status in {"flagged", "blocked"} + + def test_valid_behavioral_label(self): + agent = ComplianceAgent() + valid, msg = agent.check_behavioral_label("accumulation_like", []) + assert valid is True + + def test_invalid_behavioral_label(self): + agent = ComplianceAgent() + valid, msg = agent.check_behavioral_label("blackrock_buying", []) + assert valid is False + + def test_add_limitations_appends_statement(self): + text = "The order book shows accumulation-like behavior." + result = ComplianceAgent.add_limitations(text) + assert "level 2 data does not reveal" in result.lower() + + def test_add_limitations_does_not_duplicate(self): + text = "Level 2 data does not reveal the actual identity of market participants." + result = ComplianceAgent.add_limitations(text) + count = result.lower().count("level 2 data does not reveal") + assert count == 1 + + +class TestDecisionParliament: + def _make_reports(self, ob_bias="neutral", ob_conf=0.4, regime="ranging", + inst_label="neutral", inst_prob=0.3, risk_status="clear", + risk_veto=False, compliance_status="approved"): + from src.agents.models import ( + OrderBookAnalystReport, MarketDNAReport, InstitutionalFootprintReport, + RiskGovernorReport, ComplianceReport, + ) + import time + ts = time.time_ns() + ob = OrderBookAnalystReport( + symbol="TEST", timestamp_ns=ts, + directional_bias=ob_bias, bias_confidence=ob_conf, + key_signals=[], absorption_score=0.2, sweep_intensity=0.1, + spoof_like_score=0.1, iceberg_like_score=0.1, + spread_multiple=1.0, book_imbalance=0.1, + ) + dna = MarketDNAReport( + symbol="TEST", timestamp_ns=ts, + regime=regime, regime_confidence=0.5, + supporting_signals=[], regime_stability=0.7, + ) + inst = InstitutionalFootprintReport( + symbol="TEST", timestamp_ns=ts, + behavioral_label=inst_label, probability=inst_prob, + confidence_label="medium", reasoning=[], limitations="test", + ) + risk = RiskGovernorReport( + symbol="TEST", timestamp_ns=ts, + risk_status=risk_status, risk_score=0.1, + veto=risk_veto, veto_reasons=["test veto"] if risk_veto else [], + spread_ok=True, data_quality_ok=True, + model_confidence_ok=True, news_lockout=False, + ) + comp = ComplianceReport( + symbol="TEST", timestamp_ns=ts, + status=compliance_status, violations=[], warnings=[], + ) + return ob, dna, inst, risk, comp + + def test_risk_veto_blocks_all(self): + ob, dna, inst, risk, comp = self._make_reports(risk_veto=True, risk_status="veto") + p = DecisionParliament() + result = p.deliberate(ob, dna, inst, risk, comp) + assert result.disposition == "risk_veto" + + def test_compliance_block(self): + ob, dna, inst, risk, comp = self._make_reports(compliance_status="blocked") + p = DecisionParliament() + result = p.deliberate(ob, dna, inst, risk, comp) + assert result.disposition == "blocked" + + def test_data_insufficient(self): + from src.agents.models import RiskGovernorReport + import time + ob, dna, inst, _, comp = self._make_reports() + bad_risk = RiskGovernorReport( + symbol="TEST", timestamp_ns=time.time_ns(), + risk_status="veto", risk_score=0.9, + veto=False, veto_reasons=[], + spread_ok=True, data_quality_ok=False, + model_confidence_ok=True, news_lockout=False, + ) + p = DecisionParliament() + result = p.deliberate(ob, dna, inst, bad_risk, comp) + assert result.disposition == "data_insufficient" + + def test_paper_trade_approved_with_strong_signals(self): + ob, dna, inst, risk, comp = self._make_reports( + ob_bias="bullish", ob_conf=0.70, + regime="accumulation", + inst_label="accumulation_like", inst_prob=0.68, + risk_status="clear", + ) + p = DecisionParliament() + result = p.deliberate(ob, dna, inst, risk, comp) + assert result.disposition in {"paper_trade_approved", "watchlist"} + + def test_result_has_limitations(self): + ob, dna, inst, risk, comp = self._make_reports() + p = DecisionParliament() + result = p.deliberate(ob, dna, inst, risk, comp) + assert result.limitations != "" diff --git a/tests/test_data_models.py b/tests/test_data_models.py new file mode 100644 index 0000000..ee94cca --- /dev/null +++ b/tests/test_data_models.py @@ -0,0 +1,79 @@ +""" +tests/test_data_models.py – Unit tests for canonical data models. +""" +import time +import pytest +from src.data_intake.models import Quote, L2Level, OrderBookSnapshot, Trade, FeedHealthReport + + +def make_snapshot(bids=None, asks=None): + bids = bids or [L2Level(price=100.0, size=500), L2Level(price=99.9, size=300)] + asks = asks or [L2Level(price=100.1, size=400), L2Level(price=100.2, size=200)] + return OrderBookSnapshot(symbol="TEST", bids=bids, asks=asks, feed="test") + + +class TestQuote: + def test_spread(self): + q = Quote("TEST", bid_price=100.0, bid_size=100, ask_price=100.1, ask_size=200) + assert abs(q.spread - 0.1) < 1e-9 + + def test_mid_price(self): + q = Quote("TEST", bid_price=100.0, bid_size=100, ask_price=100.2, ask_size=100) + assert abs(q.mid_price - 100.1) < 1e-9 + + def test_microprice_pulls_toward_thin_side(self): + # Ask is thinner → microprice should be above mid + q = Quote("TEST", bid_price=100.0, bid_size=1000, ask_price=100.2, ask_size=100) + assert q.microprice > q.mid_price + + +class TestOrderBookSnapshot: + def test_spread(self): + snap = make_snapshot() + assert abs(snap.spread - 0.1) < 1e-9 + + def test_mid_price(self): + snap = make_snapshot() + assert abs(snap.mid_price - 100.05) < 1e-9 + + def test_total_bid_depth(self): + snap = make_snapshot() + assert snap.total_bid_depth() == 800 + + def test_best_bid_ask(self): + snap = make_snapshot() + assert snap.best_bid.price == 100.0 + assert snap.best_ask.price == 100.1 + + def test_empty_book(self): + snap = OrderBookSnapshot(symbol="TEST", bids=[], asks=[], feed="test") + assert snap.best_bid is None + assert snap.best_ask is None + + +class TestTrade: + def test_notional(self): + t = Trade("TEST", price=100.0, size=500, side="buy") + assert t.notional == 50000.0 + + def test_is_buy_aggressor(self): + t = Trade("TEST", price=100.0, size=100, side="buy") + assert t.is_buy_aggressor is True + t2 = Trade("TEST", price=100.0, size=100, side="sell") + assert t2.is_buy_aggressor is False + + +class TestFeedHealthReport: + def test_healthy(self): + h = FeedHealthReport( + feed="test", symbol="TEST", quality_score=0.95, latency_ms=50, + missing_ticks=0, stale_quote=False, bad_ticks=0, gap_detected=False, + ) + assert h.is_healthy is True + + def test_stale_is_unhealthy(self): + h = FeedHealthReport( + feed="test", symbol="TEST", quality_score=0.95, latency_ms=50, + missing_ticks=0, stale_quote=True, bad_ticks=0, gap_detected=False, + ) + assert h.is_healthy is False diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 0000000..379e951 --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,82 @@ +""" +tests/test_features.py – Unit tests for the Feature Engineering Layer. +""" +import math +import pytest +from src.data_intake.models import L2Level, OrderBookSnapshot, Trade +from src.order_book.engine import OrderBookEngine +from src.features.engineer import FeatureEngineer + + +def make_state(bid_size=500, ask_size=400, mid=100.0): + engine = OrderBookEngine(depth_levels=5) + bids = [L2Level(price=round(mid - 0.05 - i * 0.01, 2), size=bid_size) for i in range(5)] + asks = [L2Level(price=round(mid + 0.05 + i * 0.01, 2), size=ask_size) for i in range(5)] + snap = OrderBookSnapshot(symbol="TEST", bids=bids, asks=asks, feed="test") + trades = [Trade("TEST", price=mid, size=100, side="buy")] + return engine.process(snap, trades), trades + + +class TestFeatureEngineer: + def setup_method(self): + self.eng = FeatureEngineer("TEST") + + def test_feature_vector_structure(self): + state, trades = make_state() + fv = self.eng.update(state, trades) + assert fv.symbol == "TEST" + assert fv.feature_version == "1.0" + assert fv.mid_price > 0 + + def test_spread_in_feature_vector(self): + state, trades = make_state() + fv = self.eng.update(state, trades) + assert fv.spread > 0 + + def test_rsi_nan_with_insufficient_history(self): + state, trades = make_state() + fv = self.eng.update(state, trades) + # Only 1 tick of history; RSI requires >14+1 data points + assert math.isnan(fv.rsi_14) + + def test_rsi_computed_after_sufficient_history(self): + state, trades = make_state() + for _ in range(20): + self.eng.update(state, trades) + fv = self.eng.update(state, trades) + assert not math.isnan(fv.rsi_14) + assert 0 <= fv.rsi_14 <= 100 + + def test_accumulation_score_range(self): + state, trades = make_state(bid_size=5000, ask_size=200) + fv = self.eng.update(state, trades) + assert 0.0 <= fv.accumulation_score <= 1.0 + + def test_distribution_score_range(self): + state, trades = make_state(bid_size=200, ask_size=5000) + fv = self.eng.update(state, trades) + assert 0.0 <= fv.distribution_score <= 1.0 + + def test_to_dict_serialisable(self): + state, trades = make_state() + fv = self.eng.update(state, trades) + d = fv.to_dict() + assert isinstance(d, dict) + assert "mid_price" in d + assert "accumulation_score" in d + + def test_vwap_approaches_mid(self): + """After many identical ticks, VWAP should converge near mid-price.""" + state, trades = make_state(mid=150.0) + for _ in range(30): + fv = self.eng.update(state, trades) + assert abs(fv.vwap - 150.0) < 1.0 + + def test_bollinger_band_width_positive(self): + """BB width should be positive after sufficient price history.""" + state, trades = make_state() + for _ in range(25): + self.eng.update(state, trades) + fv = self.eng.update(state, trades) + if not math.isnan(fv.bb_width): + assert fv.bb_width >= 0 diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py new file mode 100644 index 0000000..e012d2e --- /dev/null +++ b/tests/test_orchestrator.py @@ -0,0 +1,91 @@ +""" +tests/test_orchestrator.py – Integration tests for the full pipeline. +""" +import asyncio +import pytest +from src.data_intake.sample_feed import SampleFeedAdapter +from src.orchestrator import Orchestrator + + +class TestOrchestrator: + def setup_method(self): + self.orch = Orchestrator("AAPL") + self.feed = SampleFeedAdapter("AAPL", base_price=185.0, seed=42) + + def teardown_method(self): + self.orch.close() + + def _run_ticks(self, n=10): + snap = self.feed.get_snapshot() + for _ in range(n): + trades = self.feed._generate_trades() + snap = self.feed.get_snapshot() + result = self.orch.process(snap, trades) + return result + + def test_pipeline_runs(self): + result = self._run_ticks(5) + assert result is not None + assert result.symbol == "AAPL" + + def test_disposition_is_valid(self): + result = self._run_ticks(10) + valid = { + "research_approved", "watchlist", "paper_trade_approved", + "human_review", "rejected", "risk_veto", "data_insufficient", "blocked", + } + assert result.disposition in valid + + def test_compliance_always_present(self): + result = self._run_ticks(5) + assert result.compliance_status in {"approved", "flagged", "blocked"} + + def test_limitations_always_in_result(self): + result = self._run_ticks(5) + assert result.limitations != "" + assert "level 2 data" in result.limitations.lower() + + def test_chat_responds(self): + self._run_ticks(5) + response = self.orch.chat("help") + assert response.text != "" + + def test_chat_analyze(self): + self._run_ticks(10) + response = self.orch.chat("analyze AAPL") + assert "AAPL" in response.text or "microstructure" in response.text.lower() + + def test_chat_compliance_gate(self): + """Chatbot must not deliver responses naming institutions.""" + self._run_ticks(5) + # The compliance gate should block any attempt to inject institution names + response = self.orch.chat("is BlackRock buying AAPL?") + # The response should either be blocked or not contain the assertion + assert "blackrock is buying" not in response.text.lower() + + def test_no_live_trading_default(self): + """Execution mode must be paper by default.""" + from src.config import LIVE_MODE_ENABLED + assert LIVE_MODE_ENABLED is False + + +class TestSampleFeed: + def test_stream_async(self): + async def _run(): + feed = SampleFeedAdapter("TEST", base_price=100.0, seed=1) + count = 0 + async for snap, trades in feed.stream(max_ticks=5): + assert snap.symbol == "TEST" + assert snap.best_bid is not None + assert snap.best_ask is not None + count += 1 + return count + count = asyncio.run(_run()) + assert count == 5 + + def test_snapshot_has_depth(self): + feed = SampleFeedAdapter("TEST", base_price=100.0) + snap = feed.get_snapshot() + assert len(snap.bids) > 0 + assert len(snap.asks) > 0 + assert snap.best_bid.price < snap.best_ask.price # bid < ask (no crossed book) diff --git a/tests/test_order_book_engine.py b/tests/test_order_book_engine.py new file mode 100644 index 0000000..977ce72 --- /dev/null +++ b/tests/test_order_book_engine.py @@ -0,0 +1,97 @@ +""" +tests/test_order_book_engine.py – Unit tests for the order book processing engine. +""" +import pytest +from src.data_intake.models import L2Level, OrderBookSnapshot, Trade +from src.order_book.engine import OrderBookEngine + + +def make_snapshot(bid_price=100.0, ask_price=100.1, bid_size=500, ask_size=400, levels=5): + bids = [L2Level(price=round(bid_price - i * 0.01, 2), size=bid_size - i * 10) for i in range(levels)] + asks = [L2Level(price=round(ask_price + i * 0.01, 2), size=ask_size - i * 10) for i in range(levels)] + return OrderBookSnapshot(symbol="TEST", bids=bids, asks=asks, feed="test", sequence=1) + + +class TestOrderBookEngine: + def setup_method(self): + self.engine = OrderBookEngine(depth_levels=5) + + def test_basic_processing(self): + snap = make_snapshot() + result = self.engine.process(snap, []) + assert result.symbol == "TEST" + assert result.mid_price > 0 + assert result.spread > 0 + + def test_book_imbalance_bullish(self): + """When bid depth >> ask depth, imbalance should be positive.""" + snap = OrderBookSnapshot( + symbol="TEST", + bids=[L2Level(price=100.0, size=5000)], + asks=[L2Level(price=100.1, size=100)], + feed="test", + ) + result = self.engine.process(snap, []) + assert result.book_imbalance > 0 + + def test_book_imbalance_bearish(self): + """When ask depth >> bid depth, imbalance should be negative.""" + snap = OrderBookSnapshot( + symbol="TEST", + bids=[L2Level(price=100.0, size=100)], + asks=[L2Level(price=100.1, size=5000)], + feed="test", + ) + result = self.engine.process(snap, []) + assert result.book_imbalance < 0 + + def test_stacking_detection(self): + """Abnormally large best bid should trigger stacking signal.""" + bids = [L2Level(price=100.0 - i * 0.01, size=10000 if i == 0 else 100) for i in range(5)] + asks = [L2Level(price=100.1 + i * 0.01, size=200) for i in range(5)] + snap = OrderBookSnapshot(symbol="TEST", bids=bids, asks=asks, feed="test") + result = self.engine.process(snap, []) + assert result.stacking_bid is True + + def test_sweep_intensity_with_multilevel_trades(self): + """Trades at multiple price levels should trigger sweep signal.""" + snap = make_snapshot() + trades = [ + Trade("TEST", price=100.1 + i * 0.01, size=200, side="buy") + for i in range(5) + ] + result = self.engine.process(snap, trades) + assert result.sweep_intensity > 0 + + def test_signals_list_not_empty(self): + snap = make_snapshot() + result = self.engine.process(snap, []) + assert len(result.signals) > 0 + + def test_spread_multiple_no_history(self): + """With no spread history, spread_multiple should be 1.0.""" + snap = make_snapshot() + result = self.engine.process(snap, []) + # First tick: spread_multiple equals spread / itself = 1.0 (approx) + assert result.spread_multiple > 0 + + def test_volume_analysis_buy_side(self): + snap = make_snapshot() + trades = [Trade("TEST", price=100.1, size=500, side="buy") for _ in range(5)] + result = self.engine.process(snap, trades) + # More buys → BSR should be > 0.5 + assert result.rolling_buy_sell_ratio > 0 + + def test_microprice_direction(self): + """When bid is larger, microprice should pull toward ask (thin side).""" + snap = OrderBookSnapshot( + symbol="TEST", + bids=[L2Level(price=100.0, size=2000)], + asks=[L2Level(price=100.2, size=200)], + feed="test", + ) + result = self.engine.process(snap, []) + # Microprice should be closer to ask (200 ask size → more weight to bid price... + # actually microprice = (bid*ask_size + ask*bid_size)/(bid_size+ask_size)) + # = (100.0*200 + 100.2*2000) / 2200 = (20000+200400)/2200 = 220400/2200 = 100.18 + assert result.microprice > 100.0 From f63ddf92171f8762b0f1baad6f568ae838794d18 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:24:08 +0000 Subject: [PATCH 2/4] Fix code review items: move math imports to top level, fix README heading, fix DATABASE_URL format --- .env.example | 2 +- README.md | 2 +- src/dashboard/app.py | 9 +++++---- src/order_book/engine.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index bf00a20..8dd71d7 100644 --- a/.env.example +++ b/.env.example @@ -13,7 +13,7 @@ OPENAI_API_KEY= ANTHROPIC_API_KEY= # ── Storage ─────────────────────────────────────────────────────────────────── -DATABASE_URL=******localhost:5432/microstructure +DATABASE_URL=sqlite:///data/microstructure.db # or ******localhost:5432/microstructure REDIS_URL=redis://localhost:6379/0 # ── System Defaults ─────────────────────────────────────────────────────────── diff --git a/README.md b/README.md index f9bf5fd..429a1d3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Institutional Microstructure Intelligence System -### Phase 1 — Research Prototype | Paper-Only | Compliance-Aware | Audit-Driven +## Phase 1 — Research Prototype | Paper-Only | Compliance-Aware | Audit-Driven > **"Evidence first. Risk second. Execution last."** diff --git a/src/dashboard/app.py b/src/dashboard/app.py index c7c3318..f9e4d8a 100644 --- a/src/dashboard/app.py +++ b/src/dashboard/app.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import math import time from collections import deque from threading import Thread @@ -232,13 +233,13 @@ st.subheader("📐 Technical Indicators") t1, t2, t3, t4, t5 = st.columns(5) with t1: - st.metric("RSI (14)", f"{features.rsi_14:.1f}" if features and not __import__('math').isnan(features.rsi_14) else "—") + st.metric("RSI (14)", f"{features.rsi_14:.1f}" if features and not math.isnan(features.rsi_14) else "—") with t2: - st.metric("ATR (14)", f"{features.atr_14:.4f}" if features and not __import__('math').isnan(features.atr_14) else "—") + st.metric("ATR (14)", f"{features.atr_14:.4f}" if features and not math.isnan(features.atr_14) else "—") with t3: - st.metric("MACD", f"{features.macd_line:.4f}" if features and not __import__('math').isnan(features.macd_line) else "—") + st.metric("MACD", f"{features.macd_line:.4f}" if features and not math.isnan(features.macd_line) else "—") with t4: - st.metric("BB Width", f"{features.bb_width:.4f}" if features and not __import__('math').isnan(features.bb_width) else "—") + st.metric("BB Width", f"{features.bb_width:.4f}" if features and not math.isnan(features.bb_width) else "—") with t5: st.metric("RVOL", f"{features.rvol:.2f}×" if features else "—") diff --git a/src/order_book/engine.py b/src/order_book/engine.py index 07cf1e2..3e5b564 100644 --- a/src/order_book/engine.py +++ b/src/order_book/engine.py @@ -25,6 +25,7 @@ from __future__ import annotations +import math import time from collections import deque from dataclasses import dataclass, field @@ -510,7 +511,6 @@ def _build_signals(self, **kwargs) -> list[str]: def _is_nan(v: float) -> bool: try: - import math return math.isnan(v) except Exception: return False From 2dd63c401c348d0608fd52c86018cce29e54a723 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 05:34:39 +0000 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20Phase=202A=20=E2=80=94=20WHAT/WHO/W?= =?UTF-8?q?HY=20agents,=20IPO=20microstructure,=20DeepSeek=20motivation=20?= =?UTF-8?q?engine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 12 + src/agents/decision_parliament.py | 68 +++++ src/agents/ipo_microstructure_agent.py | 177 +++++++++++++ src/agents/models.py | 52 ++++ src/agents/motivation_inference_agent.py | 266 +++++++++++++++++++ src/agents/participant_archetype_agent.py | 198 +++++++++++++++ src/audit/ledger.py | 25 +- src/chatbot/interface.py | 295 +++++++++++++++++++--- src/config.py | 13 + src/orchestrator.py | 32 ++- tests/test_agents.py | 263 +++++++++++++++++++ 11 files changed, 1364 insertions(+), 37 deletions(-) create mode 100644 src/agents/ipo_microstructure_agent.py create mode 100644 src/agents/motivation_inference_agent.py create mode 100644 src/agents/participant_archetype_agent.py diff --git a/.env.example b/.env.example index 8dd71d7..853d974 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,18 @@ ALPACA_BASE_URL=https://paper-api.alpaca.markets # paper by default OPENAI_API_KEY= ANTHROPIC_API_KEY= +# ── DeepSeek – Motivation Inference Engine (Phase 2A) ───────────────────────── +# DeepSeek-R1 is recommended over K2Think.ai for data sovereignty and open API +# compatibility. Self-host locally for full compliance control. +# Leave DEEPSEEK_API_KEY blank to use the rule-based motivation engine only. +DEEPSEEK_API_KEY= +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 # override for local deployment +DEEPSEEK_MODEL=deepseek-reasoner + +# ── IPO Symbols (Phase 2A) ───────────────────────────────────────────────────── +# Comma-separated list of IPO tickers receiving enhanced IPO microstructure analysis. +IPO_SYMBOLS=SPCX + # ── Storage ─────────────────────────────────────────────────────────────────── DATABASE_URL=sqlite:///data/microstructure.db # or ******localhost:5432/microstructure REDIS_URL=redis://localhost:6379/0 diff --git a/src/agents/decision_parliament.py b/src/agents/decision_parliament.py index 60848a5..e8cf91c 100644 --- a/src/agents/decision_parliament.py +++ b/src/agents/decision_parliament.py @@ -16,12 +16,17 @@ from __future__ import annotations +from typing import Optional + from src.agents.models import ( ComplianceReport, DecisionParliamentResult, InstitutionalFootprintReport, + IPOMicrostructureReport, MarketDNAReport, + MotivationInferenceReport, OrderBookAnalystReport, + ParticipantArchetypeReport, RiskGovernorReport, ) from src.config import COMPLIANCE_LIMITATIONS_STATEMENT @@ -39,6 +44,10 @@ def deliberate( inst_report: InstitutionalFootprintReport, risk_report: RiskGovernorReport, compliance_report: ComplianceReport, + # Phase 2A – optional extended reports + archetype_report: Optional[ParticipantArchetypeReport] = None, + motivation_report: Optional[MotivationInferenceReport] = None, + ipo_report: Optional[IPOMicrostructureReport] = None, ) -> DecisionParliamentResult: symbol = ob_report.symbol @@ -50,6 +59,7 @@ def deliberate( symbol, ts, "data_insufficient", "Data quality check failed. Feed reliability is insufficient for analysis.", ob_report, dna_report, inst_report, risk_report, compliance_report, + archetype_report, motivation_report, ipo_report, ) # ── Gate 2: Compliance block ────────────────────────────────────────── @@ -58,6 +68,7 @@ def deliberate( symbol, ts, "blocked", f"Output blocked by Compliance Agent: {'; '.join(compliance_report.violations)}", ob_report, dna_report, inst_report, risk_report, compliance_report, + archetype_report, motivation_report, ipo_report, ) # ── Gate 3: Risk veto ───────────────────────────────────────────────── @@ -66,6 +77,7 @@ def deliberate( symbol, ts, "risk_veto", f"Risk Governor veto: {'; '.join(risk_report.veto_reasons)}", ob_report, dna_report, inst_report, risk_report, compliance_report, + archetype_report, motivation_report, ipo_report, ) # ── Compute aggregate conviction ────────────────────────────────────── @@ -87,6 +99,15 @@ def deliberate( bearish_votes += 1 total_confidence += inst_report.probability + # Phase 2A: archetype vote + if archetype_report: + if archetype_report.archetype in {"institutional_accumulator", "informed_flow_proxy"}: + bullish_votes += 1 + total_confidence += archetype_report.probability + elif archetype_report.archetype in {"strategic_seller", "momentum_ignitor"}: + bearish_votes += 1 + total_confidence += archetype_report.probability + regime_supports_action = dna_report.regime in { "accumulation", "breakout", "compression", "distribution" } @@ -102,6 +123,7 @@ def deliberate( "Conflicting high-confidence signals from Order Book Analyst and " "Institutional Footprint Agent. Human review recommended.", ob_report, dna_report, inst_report, risk_report, compliance_report, + archetype_report, motivation_report, ipo_report, ) # ── Gate 5: Paper trade approval ───────────────────────────────────── @@ -119,6 +141,7 @@ def deliberate( f"Multiple agents converge on {direction} view with sufficient confidence. " f"Risk Governor clear. Paper trade approved (no live execution).", ob_report, dna_report, inst_report, risk_report, compliance_report, + archetype_report, motivation_report, ipo_report, ) # ── Gate 6: Watchlist ───────────────────────────────────────────────── @@ -128,6 +151,7 @@ def deliberate( "Signal is present but conviction is insufficient for execution. " "Setup added to watchlist for further monitoring.", ob_report, dna_report, inst_report, risk_report, compliance_report, + archetype_report, motivation_report, ipo_report, ) # ── Default: Research only ──────────────────────────────────────────── @@ -136,6 +160,7 @@ def deliberate( "Analysis complete. No actionable trade setup detected at this time. " "Research output delivered.", ob_report, dna_report, inst_report, risk_report, compliance_report, + archetype_report, motivation_report, ipo_report, ) @staticmethod @@ -149,7 +174,11 @@ def _result( inst: InstitutionalFootprintReport, risk: RiskGovernorReport, compliance: ComplianceReport, + archetype: Optional[ParticipantArchetypeReport] = None, + motivation: Optional[MotivationInferenceReport] = None, + ipo: Optional[IPOMicrostructureReport] = None, ) -> DecisionParliamentResult: + # ── Core human explanation ──────────────────────────────────────────── human_exp = ( f"Symbol: {symbol} | Regime: {dna.regime} (confidence {dna.regime_confidence:.0%}) | " f"Order book bias: {ob.directional_bias} ({ob.bias_confidence:.0%}) | " @@ -158,6 +187,40 @@ def _result( f"Disposition: {disposition.replace('_', ' ').upper()}. " f"{reasoning}" ) + + # ── Phase 2A extensions ─────────────────────────────────────────────── + arch_label: Optional[str] = None + arch_prob: Optional[float] = None + motiv_primary: Optional[str] = None + motiv_conf: Optional[str] = None + ipo_phase: Optional[str] = None + + if archetype: + arch_label = archetype.archetype + arch_prob = archetype.probability + human_exp += ( + f" | WHO: {archetype.archetype.replace('_', ' ')} " + f"({archetype.probability:.0%}, {archetype.confidence_label} confidence)" + ) + + if motivation: + motiv_primary = motivation.motivation_primary + motiv_conf = motivation.motivation_confidence + human_exp += ( + f" | WHY: {motivation.motivation_primary.replace('_', ' ')} " + f"(confidence: {motivation.motivation_confidence}, engine: {motivation.engine_used})" + ) + + if ipo: + ipo_phase = ipo.price_discovery_phase + human_exp += ( + f" | IPO Phase: {ipo.price_discovery_phase.replace('_', ' ')}" + ) + if ipo.stabilisation_agent_likely: + human_exp += " [stabilisation activity detected]" + if ipo.greenshoe_activity_likely: + human_exp += " [greenshoe-like activity]" + return DecisionParliamentResult( symbol=symbol, timestamp_ns=ts, @@ -173,4 +236,9 @@ def _result( compliance_status=compliance.status, human_explanation=human_exp, limitations=COMPLIANCE_LIMITATIONS_STATEMENT, + participant_archetype=arch_label, + participant_archetype_prob=arch_prob, + motivation_primary=motiv_primary, + motivation_confidence=motiv_conf, + ipo_phase=ipo_phase, ) diff --git a/src/agents/ipo_microstructure_agent.py b/src/agents/ipo_microstructure_agent.py new file mode 100644 index 0000000..9847050 --- /dev/null +++ b/src/agents/ipo_microstructure_agent.py @@ -0,0 +1,177 @@ +""" +agents/ipo_microstructure_agent.py – IPO Microstructure Agent (Phase 2A). + +Detects IPO-specific microstructure signals: + - Price discovery phase classification + - Stabilisation agent activity patterns + - Greenshoe (over-allotment option) exercise-like behaviour + - Lock-up expiry proximity signals + - Underwriter price support patterns + +These signals are observable from Level 2 data alone. Named entity claims +(e.g., 'the underwriter is Goldman Sachs') are prohibited. + +IPO Phases: + pre_open – pre-market, indicative only + price_discovery – first 30–90 minutes of trading; wide spread, high volume volatility + stabilisation – potential underwriter support below IPO price + post_stabilisation – stabilisation period ended + normal_trading – typical secondary-market microstructure +""" + +from __future__ import annotations + +from src.config import COMPLIANCE_LIMITATIONS_STATEMENT, IPO_SYMBOLS +from src.features.engineer import FeatureVector +from src.order_book.engine import OrderBookState +from src.agents.models import IPOMicrostructureReport + + +class IPOMicrostructureAgent: + """ + IPO-specific microstructure signal detector. + + Identifies observable patterns associated with IPO price dynamics. + All claims are behavioural — no named participant assertions. + """ + + def analyse( + self, + state: OrderBookState, + features: FeatureVector, + ) -> IPOMicrostructureReport: + is_ipo = state.symbol.upper() in {s.upper() for s in IPO_SYMBOLS} + signals: list[str] = [] + + phase = self._classify_phase(state, features, signals) + greenshoe = self._detect_greenshoe_like(state, features, signals) + stabilisation = self._detect_stabilisation(state, features, signals) + lock_up = self._detect_lock_up_proximity(state, features, signals) + + if not signals: + signals.append("No IPO-specific microstructure signals detected.") + + conf = "high" if len(signals) >= 3 else "medium" if len(signals) >= 1 else "low" + + return IPOMicrostructureReport( + symbol=state.symbol, + timestamp_ns=state.timestamp_ns, + is_ipo_symbol=is_ipo, + price_discovery_phase=phase, + greenshoe_activity_likely=greenshoe, + stabilisation_agent_likely=stabilisation, + lock_up_proximity_signal=lock_up, + ipo_specific_signals=signals, + confidence_label=conf, + ) + + @staticmethod + def _classify_phase( + state: OrderBookState, + fv: FeatureVector, + signals: list[str], + ) -> str: + """ + Infer approximate IPO trading phase from microstructure signals. + """ + # Wide spread + extreme volume = price discovery + if state.spread_multiple > 3.0 and fv.rvol > 4.0: + signals.append( + f"Extreme spread ({state.spread_multiple:.1f}×) and relative volume " + f"({fv.rvol:.1f}×) — consistent with IPO price discovery phase." + ) + return "price_discovery" + + # Bid absorption below reference + tight spread on bid side = stabilisation + if ( + state.absorption_score > 0.6 + and fv.vwap_deviation < -0.005 + and not state.stacking_ask + ): + signals.append( + "Consistent bid absorption below VWAP without ask stacking — " + "pattern consistent with price stabilisation activity." + ) + return "stabilisation" + + # Moderate spread, returning to normal depth + if state.spread_multiple < 1.5 and fv.rvol < 2.0: + return "normal_trading" + + # Default for elevated but non-extreme conditions + if state.spread_multiple > 1.5 and fv.rvol > 2.0: + return "post_stabilisation" + + return "normal_trading" + + @staticmethod + def _detect_greenshoe_like( + state: OrderBookState, + fv: FeatureVector, + signals: list[str], + ) -> bool: + """ + Detect patterns consistent with greenshoe option exercise. + Greenshoe typically creates sustained bid support just below IPO price. + """ + # Strong bid absorption + replenishment at a consistent level without price advance + if ( + state.absorption_score > 0.55 + and state.bid_replenishment_rate > 0.40 + and fv.vwap_deviation < 0.002 + and fv.order_flow_imbalance > 0.10 + ): + signals.append( + "Sustained bid absorption with high replenishment rate near reference price — " + "pattern is consistent with over-allotment support activity (greenshoe-like)." + ) + return True + return False + + @staticmethod + def _detect_stabilisation( + state: OrderBookState, + fv: FeatureVector, + signals: list[str], + ) -> bool: + """ + Detect patterns consistent with underwriter price stabilisation. + Stabilisation creates mechanical bid support at or near the IPO offer price. + """ + if ( + state.absorption_score > 0.65 + and fv.vwap_deviation < -0.003 + and fv.accumulation_score > 0.45 + and state.stacking_bid + ): + signals.append( + "Large bid stacking + absorption below VWAP — observable pattern is consistent " + "with stabilisation-agent-like support behaviour." + ) + return True + return False + + @staticmethod + def _detect_lock_up_proximity( + state: OrderBookState, + fv: FeatureVector, + signals: list[str], + ) -> bool: + """ + Detect order flow patterns that may coincide with lock-up expiry proximity. + These are purely behavioural signals — calendar data is not available at runtime + without a data feed. The signal is a pattern flag only. + """ + # Distribution pressure + growing relative volume = potential insider/employee selling + if ( + fv.distribution_score > 0.55 + and fv.rvol > 2.5 + and fv.order_flow_imbalance < -0.15 + ): + signals.append( + f"Distribution score ({fv.distribution_score:.2f}) with elevated relative volume " + f"({fv.rvol:.1f}×) and negative OFI — cross-reference with IPO lock-up calendar. " + "Pattern may be consistent with post-lock-up selling pressure." + ) + return True + return False diff --git a/src/agents/models.py b/src/agents/models.py index 5a97957..9c1abd7 100644 --- a/src/agents/models.py +++ b/src/agents/models.py @@ -72,6 +72,52 @@ class ComplianceReport: warnings: list[str] +@dataclass +class ParticipantArchetypeReport: + """WHO layer — behavioural archetype classification (Phase 2A).""" + symbol: str + timestamp_ns: int + archetype: str # institutional_accumulator | retail_sentiment_buyer | + # algorithmic_market_maker | momentum_ignitor | + # strategic_seller | informed_flow_proxy | unknown_mixed + probability: float # 0.0 – 1.0 + confidence_label: str # high | medium | low | insufficient_data + secondary_archetype: Optional[str] + secondary_probability: float + evidence: list[str] + limitations: str # always populated – compliance requirement + + +@dataclass +class IPOMicrostructureReport: + """IPO-specific microstructure signals (Phase 2A).""" + symbol: str + timestamp_ns: int + is_ipo_symbol: bool + price_discovery_phase: str # pre_open | price_discovery | stabilisation | + # post_stabilisation | normal_trading + greenshoe_activity_likely: bool + stabilisation_agent_likely: bool + lock_up_proximity_signal: bool # within 30 days of lock-up expiry + ipo_specific_signals: list[str] + confidence_label: str + + +@dataclass +class MotivationInferenceReport: + """WHY layer — motivation taxonomy inference (Phase 2A).""" + symbol: str + timestamp_ns: int + motivation_primary: str # see MOTIVATION_TAXONOMY in motivation_inference_agent.py + motivation_confidence: str # high | medium | low + motivation_evidence: list[str] + motivation_alternative: str + archetype_context: str # the WHO archetype that drove this inference + reasoning_trace: str # rule-based narrative or DeepSeek reasoning trace + engine_used: str # "rule_based" | "deepseek" + compliance_cleared: bool + + @dataclass class DecisionParliamentResult: symbol: str @@ -89,4 +135,10 @@ class DecisionParliamentResult: compliance_status: str human_explanation: str limitations: str + # Phase 2A extended fields (optional – None when agents not yet wired) + participant_archetype: Optional[str] = None + participant_archetype_prob: Optional[float] = None + motivation_primary: Optional[str] = None + motivation_confidence: Optional[str] = None + ipo_phase: Optional[str] = None timestamp_ms: int = field(default_factory=lambda: int(time.time() * 1000)) diff --git a/src/agents/motivation_inference_agent.py b/src/agents/motivation_inference_agent.py new file mode 100644 index 0000000..ea41f5e --- /dev/null +++ b/src/agents/motivation_inference_agent.py @@ -0,0 +1,266 @@ +""" +agents/motivation_inference_agent.py – Motivation Inference Agent (Phase 2A). + +WHY layer: infers the most likely economic motivation behind the dominant +participant archetype observed in the order flow. + +Two-mode engine: + 1. Rule-based (default) — deterministic, requires no API keys, fully auditable. + 2. DeepSeek-R1 enhanced — if DEEPSEEK_API_KEY is set, the rule-based output is + enriched with a DeepSeek reasoning trace via the OpenAI-compatible API. + Falls back to rule-based silently if the API call fails. + +MOTIVATION TAXONOMY +─────────────────── +ACCUMULATION: + position_building_ahead_of_catalyst – pre-catalyst accumulation + index_rebalancing – mandatory passive rebalancing + mandate_driven_allocation – pension / endowment / passive fund mandate + arbitrage_convergence – spread/convergence play + informed_flow_proxy – corroborated by public SEC filings only + +DISTRIBUTION: + lock_up_expiry_selling – approaching lock-up expiry + stop_loss_exit – involuntary risk exit + profit_taking_resistance – technical profit-taking + risk_reduction_macro_event – de-risking ahead of macro catalyst + tax_loss_harvesting – seasonal / year-end + short_entry_negative_thesis – directional short + +NEUTRAL / LIQUIDITY: + market_making_neutral – no directional bias + portfolio_rebalancing_neutral – size-neutral rebalancing + options_hedging_delta_adjustment – delta hedge + +UNKNOWN: + unknown – insufficient evidence +""" + +from __future__ import annotations + +from src.config import ( + COMPLIANCE_LIMITATIONS_STATEMENT, + DEEPSEEK_API_KEY, + DEEPSEEK_BASE_URL, + DEEPSEEK_MODEL, +) +from src.features.engineer import FeatureVector +from src.order_book.engine import OrderBookState +from src.agents.models import MotivationInferenceReport, ParticipantArchetypeReport + + +# Full taxonomy set used for validation +MOTIVATION_TAXONOMY = { + # Accumulation + "position_building_ahead_of_catalyst", + "index_rebalancing", + "mandate_driven_allocation", + "arbitrage_convergence", + "informed_flow_proxy", + # Distribution + "lock_up_expiry_selling", + "stop_loss_exit", + "profit_taking_resistance", + "risk_reduction_macro_event", + "tax_loss_harvesting", + "short_entry_negative_thesis", + # Neutral + "market_making_neutral", + "portfolio_rebalancing_neutral", + "options_hedging_delta_adjustment", + # Unknown + "unknown", +} + +# Archetype → motivation mapping (rule-based prior) +_ARCHETYPE_MOTIVATION_MAP: dict[str, tuple[str, str]] = { + "institutional_accumulator": ("position_building_ahead_of_catalyst", "mandate_driven_allocation"), + "retail_sentiment_buyer": ("position_building_ahead_of_catalyst", "stop_loss_exit"), + "algorithmic_market_maker": ("market_making_neutral", "options_hedging_delta_adjustment"), + "momentum_ignitor": ("short_entry_negative_thesis", "profit_taking_resistance"), + "strategic_seller": ("profit_taking_resistance", "risk_reduction_macro_event"), + "informed_flow_proxy": ("position_building_ahead_of_catalyst", "informed_flow_proxy"), + "unknown_mixed": ("unknown", "unknown"), +} + + +class MotivationInferenceAgent: + """ + WHY-layer inference. + + Call analyse() after ParticipantArchetypeAgent to obtain a motivation + taxonomy label, confidence score, and supporting evidence. + + If DEEPSEEK_API_KEY is set in config, the output is enriched with a + DeepSeek-R1 reasoning trace. The rule-based result is always produced + first so that the system is never dependent on external API availability. + """ + + def analyse( + self, + state: OrderBookState, + features: FeatureVector, + archetype_report: ParticipantArchetypeReport, + ) -> MotivationInferenceReport: + # Always produce rule-based result first + primary, alt, evidence, confidence, trace = self._rule_based( + state, features, archetype_report + ) + engine = "rule_based" + + # Optionally enrich with DeepSeek if API key is available + if DEEPSEEK_API_KEY: + try: + ds_trace = self._deepseek_enrich(state, features, archetype_report, primary, evidence) + trace = ds_trace + engine = "deepseek" + except Exception: + pass # Silently fall back to rule-based + + return MotivationInferenceReport( + symbol=state.symbol, + timestamp_ns=state.timestamp_ns, + motivation_primary=primary, + motivation_confidence=confidence, + motivation_evidence=evidence, + motivation_alternative=alt, + archetype_context=archetype_report.archetype, + reasoning_trace=trace, + engine_used=engine, + compliance_cleared=True, + ) + + @staticmethod + def _rule_based( + state: OrderBookState, + fv: FeatureVector, + arch: ParticipantArchetypeReport, + ) -> tuple[str, str, list[str], str, str]: + """ + Deterministic rule engine. Returns: + (primary_motivation, alt_motivation, evidence_list, confidence, trace) + """ + archetype = arch.archetype + primary, alt = _ARCHETYPE_MOTIVATION_MAP.get(archetype, ("unknown", "unknown")) + evidence: list[str] = list(arch.evidence) # carry forward archetype evidence + trace_parts: list[str] = [f"Archetype: {archetype} (p={arch.probability:.2f})."] + + # ── Refine primary motivation using contextual signals ──────────────── + + if archetype == "institutional_accumulator": + if fv.rvol > 3.0: + primary = "position_building_ahead_of_catalyst" + evidence.append(f"Relative volume spike ({fv.rvol:.1f}×) consistent with catalyst-driven accumulation.") + trace_parts.append("Elevated RVOL → catalyst-driven accumulation hypothesis elevated.") + elif fv.accumulation_score > 0.6 and fv.order_flow_imbalance < 0.1: + primary = "mandate_driven_allocation" + evidence.append("Quiet, low-urgency accumulation pattern consistent with mandate-driven flow.") + trace_parts.append("Low urgency + steady accumulation → mandate allocation.") + else: + evidence.append("Standard accumulation pattern — catalyst or mandate motivation probable.") + + elif archetype == "retail_sentiment_buyer": + if fv.rvol > 2.5: + evidence.append(f"High relative volume ({fv.rvol:.1f}×) amplifies retail FOMO hypothesis.") + trace_parts.append("RVOL > 2.5× with retail signature → FOMO motivation elevated.") + elif fv.rolling_buy_sell_ratio > 1.8: + evidence.append(f"Buy/sell ratio of {fv.rolling_buy_sell_ratio:.1f}× is typical of sentiment-driven buying.") + + elif archetype == "algorithmic_market_maker": + if abs(fv.bid_ask_imbalance) < 0.05: + evidence.append("Near-zero book imbalance confirms neutral market-making stance.") + trace_parts.append("Near-zero imbalance → market-making neutral confirmed.") + + elif archetype == "momentum_ignitor": + if state.spoof_like_score > 0.6: + primary = "short_entry_negative_thesis" + evidence.append(f"Spoof-like score {state.spoof_like_score:.2f} consistent with directional short entry.") + trace_parts.append("High spoof score → short-entry / momentum manipulation hypothesis.") + elif state.sweep_intensity > 0.8: + evidence.append(f"Sweep intensity {state.sweep_intensity:.2f} — stop-cluster triggering pattern.") + + elif archetype == "strategic_seller": + if fv.distribution_score > 0.6: + if fv.vwap_deviation < -0.005: + primary = "stop_loss_exit" + evidence.append("Selling below VWAP with high distribution score — involuntary stop-loss exit pattern.") + trace_parts.append("Below-VWAP distribution → stop-loss exit hypothesis elevated.") + else: + primary = "profit_taking_resistance" + evidence.append("Selling into resistance with high distribution score — profit-taking pattern.") + trace_parts.append("At-resistance distribution → profit-taking hypothesis elevated.") + if fv.rvol > 3.0: + alt = "risk_reduction_macro_event" + evidence.append(f"Volume spike ({fv.rvol:.1f}×) with selling pressure — macro risk-reduction possible.") + + elif archetype == "informed_flow_proxy": + evidence.append( + "Pattern is consistent with informed positioning. " + "Cross-reference SEC EDGAR 13F / Form 4 filings before drawing conclusions." + ) + trace_parts.append("Informed flow proxy — public disclosure corroboration required.") + + # Confidence based on archetype confidence + evidence count + if arch.confidence_label == "high" and len(evidence) >= 2: + confidence = "high" + elif arch.confidence_label in {"high", "medium"} and len(evidence) >= 1: + confidence = "medium" + else: + confidence = "low" + + if not evidence: + evidence.append("Insufficient observable signals to determine motivation with confidence.") + confidence = "low" + + trace = " | ".join(trace_parts) if trace_parts else "No rule triggers fired." + return primary, alt, evidence, confidence, trace + + @staticmethod + def _deepseek_enrich( + state: OrderBookState, + fv: FeatureVector, + arch: ParticipantArchetypeReport, + primary_motivation: str, + evidence: list[str], + ) -> str: + """ + Send a structured prompt to DeepSeek-R1 via OpenAI-compatible API + and return the reasoning trace. Never replaces compliance-cleared + rule-based output — only enriches the reasoning narrative. + """ + import openai # imported lazily to avoid hard dependency when not needed + + client = openai.OpenAI( + api_key=DEEPSEEK_API_KEY, + base_url=DEEPSEEK_BASE_URL, + ) + + prompt = ( + f"You are an institutional market microstructure analyst. " + f"You NEVER identify named market participants. " + f"You speak in probabilistic, compliance-safe language.\n\n" + f"INSTRUMENT: {state.symbol}\n" + f"PARTICIPANT ARCHETYPE: {arch.archetype} (probability: {arch.probability:.0%}, " + f"confidence: {arch.confidence_label})\n" + f"RULE-BASED MOTIVATION: {primary_motivation}\n" + f"EVIDENCE:\n" + "\n".join(f" - {e}" for e in evidence) + "\n\n" + f"MARKET CONTEXT:\n" + f" Order flow imbalance: {fv.order_flow_imbalance:+.3f}\n" + f" Relative volume: {fv.rvol:.2f}×\n" + f" VWAP deviation: {fv.vwap_deviation:+.4f}\n" + f" Absorption score: {state.absorption_score:.2f}\n" + f" Sweep intensity: {state.sweep_intensity:.2f}\n" + f" Spoof-like score: {state.spoof_like_score:.2f}\n\n" + f"Task: In 2–3 sentences, provide an institutional-grade reasoning narrative " + f"explaining WHY this archetype is most likely placing orders now. " + f"Do NOT name any institution. Use probabilistic language. " + f"State confidence level. Mention any alternative motivation if plausible." + ) + + response = client.chat.completions.create( + model=DEEPSEEK_MODEL, + messages=[{"role": "user", "content": prompt}], + max_tokens=300, + temperature=0.2, + ) + return response.choices[0].message.content.strip() diff --git a/src/agents/participant_archetype_agent.py b/src/agents/participant_archetype_agent.py new file mode 100644 index 0000000..0fc8eb1 --- /dev/null +++ b/src/agents/participant_archetype_agent.py @@ -0,0 +1,198 @@ +""" +agents/participant_archetype_agent.py – Participant Archetype Agent (Phase 2A). + +WHO layer: classifies the behavioural archetype of market participants observed +in Level 2 order-flow data. + +COMPLIANCE REQUIREMENT (HARD): + This agent NEVER identifies, names, or implies the identity of any + specific market participant, institution, firm, fund, or individual. + All outputs are probabilistic behavioural archetype labels with mandatory + limitations statements. + +Archetypes (observable behaviour signatures only): + institutional_accumulator – iceberg orders, VWAP-anchored absorption, low urgency + retail_sentiment_buyer – market orders at open, round-number clusters, high spread tolerance + algorithmic_market_maker – tight bid/ask cycling, rapid quote refresh, mean-reversion + momentum_ignitor – sweep patterns, spoof-then-pull, volume spike + fast retrace + strategic_seller – ask stacking, sweep exhaustion, dark-print divergence + informed_flow_proxy – pre-catalyst accumulation, size concentration at key levels + (requires corroborating public data; never asserts insider knowledge) + unknown_mixed – no dominant pattern +""" + +from __future__ import annotations + +from src.config import COMPLIANCE_LIMITATIONS_STATEMENT +from src.features.engineer import FeatureVector +from src.order_book.engine import OrderBookState +from src.agents.models import ParticipantArchetypeReport + + +# Archetype labels – exhaustive permitted set +ARCHETYPES = { + "institutional_accumulator", + "retail_sentiment_buyer", + "algorithmic_market_maker", + "momentum_ignitor", + "strategic_seller", + "informed_flow_proxy", + "unknown_mixed", +} + + +class ParticipantArchetypeAgent: + """ + WHO-layer inference. Scores seven mutually exclusive behavioural archetypes + using observable Level 2 signals. Returns the dominant archetype plus a + secondary archetype when two patterns are in close competition. + """ + + def analyse( + self, + state: OrderBookState, + features: FeatureVector, + ) -> ParticipantArchetypeReport: + scores, evidence = self._score_archetypes(state, features) + + # Sort by score descending + ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True) + primary_arch, primary_prob = ranked[0] + secondary_arch, secondary_prob = ranked[1] + + # Normalise to keep within [0, 1] + primary_prob = round(min(primary_prob, 1.0), 3) + secondary_prob = round(min(secondary_prob, 1.0), 3) + + # Confidence label + if primary_prob >= 0.65: + conf = "high" + elif primary_prob >= 0.45: + conf = "medium" + elif primary_prob >= 0.30: + conf = "low" + else: + primary_arch = "unknown_mixed" + conf = "insufficient_data" + + # If secondary is too close to primary, both are relevant + secondary = secondary_arch if secondary_prob >= 0.30 else None + + return ParticipantArchetypeReport( + symbol=state.symbol, + timestamp_ns=state.timestamp_ns, + archetype=primary_arch, + probability=primary_prob, + confidence_label=conf, + secondary_archetype=secondary, + secondary_probability=secondary_prob, + evidence=evidence.get(primary_arch, []), + limitations=COMPLIANCE_LIMITATIONS_STATEMENT, + ) + + @staticmethod + def _score_archetypes( + state: OrderBookState, + fv: FeatureVector, + ) -> tuple[dict[str, float], dict[str, list[str]]]: + """ + Score each archetype from 0.0 to 1.0 using observable signals. + Returns (scores_dict, evidence_dict). + """ + scores: dict[str, float] = {a: 0.0 for a in ARCHETYPES} + evidence: dict[str, list[str]] = {a: [] for a in ARCHETYPES} + + # ── Institutional Accumulator ───────────────────────────────────────── + ia = "institutional_accumulator" + if state.iceberg_like_score > 0.4: + scores[ia] += 0.20 + evidence[ia].append(f"Iceberg-like clip patterns (score {state.iceberg_like_score:.2f}).") + if state.absorption_score > 0.5: + scores[ia] += 0.15 + evidence[ia].append(f"Large-print absorption at bid (score {state.absorption_score:.2f}).") + if fv.vwap_deviation > 0.002 and fv.order_flow_imbalance > 0.15: + scores[ia] += 0.12 + evidence[ia].append("Price above VWAP with positive OFI — VWAP-anchored accumulation pattern.") + if state.bid_replenishment_rate > 0.3: + scores[ia] += 0.10 + evidence[ia].append(f"High bid replenishment rate ({state.bid_replenishment_rate:.2f}) — consistent with patient buyer.") + if fv.accumulation_score > 0.5: + scores[ia] += 0.15 + evidence[ia].append(f"Composite accumulation score elevated ({fv.accumulation_score:.2f}).") + + # ── Retail Sentiment Buyer ──────────────────────────────────────────── + rs = "retail_sentiment_buyer" + if state.sweep_intensity > 0.5 and fv.rolling_buy_sell_ratio > 1.3: + scores[rs] += 0.20 + evidence[rs].append("Market-order sweep with high buy/sell ratio — urgency consistent with retail FOMO.") + if state.spread_multiple > 2.0 and fv.order_flow_imbalance > 0.1: + scores[rs] += 0.12 + evidence[rs].append(f"Buying into wide spread ({state.spread_multiple:.1f}×) — low price sensitivity.") + if fv.rvol > 2.0 and fv.rolling_buy_sell_ratio > 1.5: + scores[rs] += 0.15 + evidence[rs].append(f"Elevated relative volume ({fv.rvol:.1f}×) with strong buy imbalance — sentiment-driven demand.") + + # ── Algorithmic Market Maker ────────────────────────────────────────── + mm = "algorithmic_market_maker" + if state.bid_replenishment_rate > 0.5 and state.ask_replenishment_rate > 0.5: + scores[mm] += 0.20 + evidence[mm].append("Simultaneous high bid and ask replenishment — symmetric quoting pattern.") + if state.spread_multiple < 1.2 and abs(fv.bid_ask_imbalance) < 0.15: + scores[mm] += 0.18 + evidence[mm].append("Tight spread with near-balanced book — market-maker neutral positioning.") + if state.spoof_like_score < 0.1 and state.iceberg_like_score < 0.1: + scores[mm] += 0.08 + evidence[mm].append("No spoof or iceberg patterns — clean two-sided quoting.") + + # ── Momentum Ignitor ───────────────────────────────────────────────── + mi = "momentum_ignitor" + if state.sweep_intensity > 0.7: + scores[mi] += 0.25 + evidence[mi].append(f"High sweep intensity ({state.sweep_intensity:.2f}) — aggressive level-taking.") + if state.spoof_like_score > 0.5: + scores[mi] += 0.20 + evidence[mi].append(f"Spoof-like activity detected (score {state.spoof_like_score:.2f}) — order placement and cancellation pattern.") + if fv.rvol > 3.0 and fv.momentum_ignition_risk > 0.5: + scores[mi] += 0.15 + evidence[mi].append("Extreme relative volume with high momentum-ignition risk score.") + + # ── Strategic Seller ───────────────────────────────────────────────── + ss = "strategic_seller" + if state.stacking_ask: + scores[ss] += 0.20 + evidence[ss].append("Ask-side stacking at resistance — controlled distribution pattern.") + if fv.distribution_score > 0.5: + scores[ss] += 0.18 + evidence[ss].append(f"Composite distribution score elevated ({fv.distribution_score:.2f}).") + if fv.vwap_deviation < -0.003 and fv.order_flow_imbalance < -0.15: + scores[ss] += 0.12 + evidence[ss].append("Price below VWAP with negative OFI — strategic selling pressure.") + if state.ask_replenishment_rate > 0.4 and not state.stacking_bid: + scores[ss] += 0.10 + evidence[ss].append("Asymmetric ask replenishment without bid support — one-sided supply.") + + # ── Informed Flow Proxy ─────────────────────────────────────────────── + # Only triggered when both accumulation AND catalyst-consistent signals are present. + # NEVER implies insider knowledge — requires public corroboration. + ip = "informed_flow_proxy" + if ( + fv.accumulation_score > 0.6 + and state.iceberg_like_score > 0.5 + and fv.institutional_footprint_prob > 0.6 + ): + scores[ip] += 0.25 + evidence[ip].append( + "High accumulation + iceberg score + institutional footprint probability — " + "pattern consistent with informed flow (public corroboration required)." + ) + if fv.breakout_confirmation_score > 0.6 and state.absorption_score > 0.6: + scores[ip] += 0.12 + evidence[ip].append("Breakout confirmation with strong absorption — pre-catalyst positioning signal.") + + # ── Fallback: unknown_mixed ─────────────────────────────────────────── + max_score = max(scores[a] for a in ARCHETYPES if a != "unknown_mixed") + if max_score < 0.20: + scores["unknown_mixed"] = 0.25 + evidence["unknown_mixed"].append("No dominant observable pattern. Mixed or ambiguous order flow.") + + return scores, evidence diff --git a/src/audit/ledger.py b/src/audit/ledger.py index 916d5ad..c4e2db1 100644 --- a/src/audit/ledger.py +++ b/src/audit/ledger.py @@ -26,7 +26,7 @@ import orjson from src.config import AUDIT_LOG_DIR -from src.agents.models import DecisionParliamentResult +from src.agents.models import DecisionParliamentResult, MotivationInferenceReport from src.data_intake.models import FeedHealthReport from src.features.engineer import FeatureVector @@ -63,9 +63,32 @@ def log_decision(self, result: DecisionParliamentResult) -> None: "compliance_status": result.compliance_status, "human_explanation": result.human_explanation, "limitations": result.limitations, + # Phase 2A extended fields + "participant_archetype": result.participant_archetype, + "participant_archetype_prob": result.participant_archetype_prob, + "motivation_primary": result.motivation_primary, + "motivation_confidence": result.motivation_confidence, + "ipo_phase": result.ipo_phase, } self._write(result.symbol, record) + def log_motivation(self, report: MotivationInferenceReport) -> None: + """Write a Phase 2A motivation inference record to the audit ledger.""" + record = { + "record_type": "motivation", + "timestamp": report.timestamp_ns, + "symbol": report.symbol, + "archetype_context": report.archetype_context, + "motivation_primary": report.motivation_primary, + "motivation_confidence": report.motivation_confidence, + "motivation_alternative": report.motivation_alternative, + "motivation_evidence": report.motivation_evidence, + "engine_used": report.engine_used, + "reasoning_trace": report.reasoning_trace, + "compliance_cleared": report.compliance_cleared, + } + self._write(report.symbol, record) + def log_health(self, health: FeedHealthReport) -> None: record = { "record_type": "data_quality", diff --git a/src/chatbot/interface.py b/src/chatbot/interface.py index c3c0f3f..16af83d 100644 --- a/src/chatbot/interface.py +++ b/src/chatbot/interface.py @@ -1,12 +1,18 @@ """ -chatbot/interface.py – Chatbot Interface Layer (Phase 1). +chatbot/interface.py – Chatbot Interface Layer (Phase 2A). The user-facing command centre. Accepts natural-language queries and routes them to the appropriate agents. Returns structured, human-readable responses. -Supported commands (Phase 1 — no live LLM required): +Supported commands (Phase 2A): "analyze {SYMBOL}" - "analyze {SYMBOL} level 2" + "who is buying / who is placing orders" + "why is there buying / selling pressure" + "what is the order flow / what is being ordered" + "is this institutional or retail" + "what is the IPO regime" + "explain the motivation" + "summarize participant activity" "is there buyer absorption" "are large sellers stacking the ask" "summarize institutional activity" @@ -30,20 +36,29 @@ ComplianceReport, DecisionParliamentResult, InstitutionalFootprintReport, + IPOMicrostructureReport, MarketDNAReport, + MotivationInferenceReport, OrderBookAnalystReport, + ParticipantArchetypeReport, RiskGovernorReport, ) from src.config import COMPLIANCE_LIMITATIONS_STATEMENT HELP_TEXT = """ -╔══════════════════════════════════════════════════════════════════╗ -║ Institutional Microstructure Intelligence — Phase 1 Chatbot ║ -╚══════════════════════════════════════════════════════════════════╝ +╔════════════════════════════════════════════════════════════════════╗ +║ Institutional Microstructure Intelligence — Phase 2A Chatbot ║ +╚════════════════════════════════════════════════════════════════════╝ Available commands: - analyze Full Level 2 microstructure analysis + analyze Full WHAT / WHO / WHY microstructure analysis + who WHO is placing orders? (participant archetype) + why WHY are they placing orders? (motivation inference) + what WHAT is being ordered? (order classification) + archetype Detailed participant archetype breakdown + motivation Detailed motivation taxonomy output + ipo IPO-specific microstructure signals absorption Is buyer absorption present at the bid? stacking Are large sellers stacking the ask? institutional Summarise institutional-style activity @@ -56,7 +71,8 @@ Notes: • All analysis is for research purposes only. • No live execution. Paper-trade mode only by default. - • Behavioural classifications are probabilistic estimates. + • Participant archetypes are probabilistic behavioural labels — not identities. + • Motivation inferences are hypotheses bounded by observable evidence. • Participant identity CANNOT be inferred from Level 2 data. """.strip() @@ -78,6 +94,12 @@ class ChatbotInterface: INTENTS: list[tuple[str, list[str]]] = [ ("analyze", ["analyze", "analysis", "level 2", "l2", "activity"]), + ("who", ["who is buying", "who is placing", "who is selling", "who is it", "who is"]), + ("why", ["why is there", "why are they", "why is buying", "why is selling", "motivation", "why"]), + ("what", ["what is being ordered", "what is being traded", "what orders", "what is the order flow"]), + ("archetype", ["archetype", "participant type", "participant archetype", "retail or institutional"]), + ("motivation", ["explain the motivation", "explain motivation", "motivation inference"]), + ("ipo", ["ipo", "ipo regime", "price discovery", "greenshoe", "lock-up", "lock up", "stabilisation"]), ("absorption", ["absorption", "buyer absorption", "absorb"]), ("stacking", ["stacking", "stack", "large sellers", "ask wall"]), ("institutional", ["institutional", "large participant", "footprint", "accumulation", "distribution"]), @@ -92,7 +114,7 @@ def parse_intent(self, user_input: str) -> tuple[str, Optional[str]]: """Extract intent and optional symbol from raw user input.""" text = user_input.lower().strip() - # Extract ticker symbol: 1–5 uppercase letters, possibly followed/preceded by space + # Extract ticker symbol: 1–5 uppercase letters symbol_match = re.search(r'\b([A-Z]{1,5})\b', user_input) symbol = symbol_match.group(1) if symbol_match else None @@ -111,10 +133,11 @@ def respond( risk_report: Optional[RiskGovernorReport] = None, compliance_report: Optional[ComplianceReport] = None, parliament_result: Optional[DecisionParliamentResult] = None, + # Phase 2A + archetype_report: Optional[ParticipantArchetypeReport] = None, + motivation_report: Optional[MotivationInferenceReport] = None, + ipo_report: Optional[IPOMicrostructureReport] = None, ) -> ChatbotResponse: - """ - Generate a human-readable response given user input and available agent reports. - """ intent, symbol = self.parse_intent(user_input) if intent == "help" or not any([ob_report, dna_report, parliament_result]): @@ -126,7 +149,20 @@ def respond( ) if intent == "analyze": - text = self._fmt_analyze(symbol, ob_report, dna_report, inst_report, risk_report, parliament_result) + text = self._fmt_analyze(symbol, ob_report, dna_report, inst_report, risk_report, + parliament_result, archetype_report, motivation_report, ipo_report) + elif intent == "who": + text = self._fmt_who(symbol, archetype_report) + elif intent == "why": + text = self._fmt_why(symbol, motivation_report) + elif intent == "what": + text = self._fmt_what(symbol, ob_report) + elif intent == "archetype": + text = self._fmt_archetype(symbol, archetype_report) + elif intent == "motivation": + text = self._fmt_motivation_detail(symbol, motivation_report) + elif intent == "ipo": + text = self._fmt_ipo(symbol, ipo_report) elif intent == "absorption": text = self._fmt_absorption(symbol, ob_report) elif intent == "stacking": @@ -166,18 +202,16 @@ def respond( @staticmethod def _fmt_analyze( - symbol, ob, dna, inst, risk, parl + symbol, ob, dna, inst, risk, parl, + archetype=None, motivation=None, ipo=None, ) -> str: lines = [ - f"{'═'*60}", - f" MICROSTRUCTURE ANALYSIS: {symbol or 'N/A'}", - f"{'═'*60}", + f"{'═'*64}", + f" MICROSTRUCTURE ANALYSIS · WHAT / WHO / WHY · {symbol or 'N/A'}", + f"{'═'*64}", ] - if dna: - lines += [ - f" Market Regime : {dna.regime.upper().replace('_', ' ')}", - f" Regime Confidence : {dna.regime_confidence:.0%}", - ] + # WHAT + lines.append(" ── WHAT is being ordered? ──────────────────────────────────") if ob: lines += [ f" Order Book Bias : {ob.directional_bias.upper()}", @@ -188,25 +222,64 @@ def _fmt_analyze( f" Sweep Intensity : {ob.sweep_intensity:.2f}", f" Spoof-like Score : {ob.spoof_like_score:.2f}", ] - if inst: + # WHO + lines.append("") + lines.append(" ── WHO is placing orders? ──────────────────────────────────") + if archetype: + lines += [ + f" Participant Type : {archetype.archetype.replace('_', ' ').upper()}", + f" Probability : {archetype.probability:.0%} [{archetype.confidence_label} confidence]", + ] + if archetype.secondary_archetype: + lines.append( + f" Secondary Type : {archetype.secondary_archetype.replace('_', ' ')} " + f"({archetype.secondary_probability:.0%})" + ) + if archetype.evidence: + lines.append(" Evidence :") + for e in archetype.evidence[:3]: + lines.append(f" • {e}") + elif inst: lines += [ - "", f" Behavioral Pattern: {inst.behavioral_label.replace('_', ' ').upper()}", f" Footprint Prob : {inst.probability:.0%} [{inst.confidence_label} confidence]", - f" Reasoning :", ] - for r in inst.reasoning[:3]: - lines.append(f" • {r}") + # WHY + lines.append("") + lines.append(" ── WHY are they placing orders? ────────────────────────────") + if motivation: + lines += [ + f" Primary Motivation: {motivation.motivation_primary.replace('_', ' ').upper()}", + f" Confidence : {motivation.motivation_confidence.upper()}", + f" Alternative : {motivation.motivation_alternative.replace('_', ' ')}", + f" Engine : {motivation.engine_used}", + ] + if motivation.motivation_evidence: + lines.append(" Motivation Evidence:") + for e in motivation.motivation_evidence[:3]: + lines.append(f" • {e}") + # IPO + if ipo and ipo.is_ipo_symbol: + lines.append("") + lines.append(" ── IPO Microstructure ──────────────────────────────────────") + lines += [ + f" IPO Phase : {ipo.price_discovery_phase.replace('_', ' ').upper()}", + f" Stabilisation : {'LIKELY' if ipo.stabilisation_agent_likely else 'Not detected'}", + f" Greenshoe-like : {'LIKELY' if ipo.greenshoe_activity_likely else 'Not detected'}", + f" Lock-up Signal : {'ACTIVE' if ipo.lock_up_proximity_signal else 'Not detected'}", + ] + # Regime + Decision + lines.append("") + if dna: + lines += [ + f" Market Regime : {dna.regime.upper().replace('_', ' ')}", + f" Regime Confidence : {dna.regime_confidence:.0%}", + ] if risk: lines += [ - "", f" Risk Status : {risk.risk_status.upper()}", f" Risk Score : {risk.risk_score:.2f}", ] - if risk.veto_reasons: - lines.append(" Risk Warnings :") - for v in risk.veto_reasons[:3]: - lines.append(f" ⚠ {v}") if parl: lines += [ "", @@ -216,7 +289,153 @@ def _fmt_analyze( lines += [ "", f" ⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}", - f"{'─'*60}", + f"{'─'*64}", + ] + return "\n".join(lines) + + @staticmethod + def _fmt_who(symbol, archetype) -> str: + if not archetype: + return "No participant archetype data available. Run 'analyze' first." + lines = [ + f"WHO Analysis — {symbol or 'N/A'}", + f"{'─'*44}", + f"Participant Archetype : {archetype.archetype.replace('_', ' ').upper()}", + f"Probability : {archetype.probability:.0%}", + f"Confidence : {archetype.confidence_label.upper()}", + ] + if archetype.secondary_archetype: + lines.append( + f"Secondary Archetype : {archetype.secondary_archetype.replace('_', ' ')} " + f"({archetype.secondary_probability:.0%})" + ) + lines.append("\nEvidence:") + for e in archetype.evidence: + lines.append(f" • {e}") + lines += ["", f"⚠️ {archetype.limitations}"] + return "\n".join(lines) + + @staticmethod + def _fmt_why(symbol, motivation) -> str: + if not motivation: + return "No motivation inference available. Run 'analyze' first." + lines = [ + f"WHY Analysis — {symbol or 'N/A'}", + f"{'─'*44}", + f"Primary Motivation : {motivation.motivation_primary.replace('_', ' ').upper()}", + f"Confidence : {motivation.motivation_confidence.upper()}", + f"Alternative : {motivation.motivation_alternative.replace('_', ' ')}", + f"Archetype Context : {motivation.archetype_context.replace('_', ' ')}", + f"Engine : {motivation.engine_used}", + "", + "Motivation Evidence:", + ] + for e in motivation.motivation_evidence: + lines.append(f" • {e}") + lines += [ + "", + "Reasoning Trace:", + f" {motivation.reasoning_trace}", + "", + f"⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}", + ] + return "\n".join(lines) + + @staticmethod + def _fmt_what(symbol, ob) -> str: + if not ob: + return "No order book data available." + lines = [ + f"WHAT Analysis — {symbol or 'N/A'}", + f"{'─'*44}", + f"Directional Bias : {ob.directional_bias.upper()}", + f"Bias Confidence : {ob.bias_confidence:.0%}", + f"Book Imbalance : {ob.book_imbalance:+.3f}", + f"Spread Multiple : {ob.spread_multiple:.1f}×", + f"Absorption Score : {ob.absorption_score:.2f} (0=none, 1=strong)", + f"Sweep Intensity : {ob.sweep_intensity:.2f} (0=none, 1=aggressive sweep)", + f"Iceberg-like Score : {ob.iceberg_like_score:.2f}", + f"Spoof-like Score : {ob.spoof_like_score:.2f}", + "", + "Active Signals:", + ] + for s in ob.key_signals: + lines.append(f" • {s}") + lines += ["", f"⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}"] + return "\n".join(lines) + + @staticmethod + def _fmt_archetype(symbol, archetype) -> str: + if not archetype: + return "No archetype data available. Run 'analyze' first." + lines = [ + f"Participant Archetype — {symbol or 'N/A'}", + f"{'─'*48}", + f"Primary Archetype : {archetype.archetype.replace('_', ' ').upper()}", + f"Probability : {archetype.probability:.0%}", + f"Confidence : {archetype.confidence_label.upper()}", + ] + if archetype.secondary_archetype: + lines += [ + f"Secondary Archetype : {archetype.secondary_archetype.replace('_', ' ')}", + f"Secondary Prob : {archetype.secondary_probability:.0%}", + ] + lines.append("\nSupporting Evidence:") + for e in archetype.evidence: + lines.append(f" • {e}") + lines += ["", f"⚠️ {archetype.limitations}"] + return "\n".join(lines) + + @staticmethod + def _fmt_motivation_detail(symbol, motivation) -> str: + if not motivation: + return "No motivation data available. Run 'analyze' first." + lines = [ + f"Motivation Inference — {symbol or 'N/A'}", + f"{'─'*48}", + f"Primary : {motivation.motivation_primary.replace('_', ' ').upper()}", + f"Confidence : {motivation.motivation_confidence.upper()}", + f"Alternative : {motivation.motivation_alternative.replace('_', ' ')}", + f"Archetype Context : {motivation.archetype_context.replace('_', ' ')}", + f"Inference Engine : {motivation.engine_used}", + f"Compliance Cleared : {'✓' if motivation.compliance_cleared else '✗'}", + "", + "Motivation Evidence:", + ] + for e in motivation.motivation_evidence: + lines.append(f" • {e}") + lines += [ + "", + "Reasoning Trace:", + f" {motivation.reasoning_trace}", + "", + f"⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}", + ] + return "\n".join(lines) + + @staticmethod + def _fmt_ipo(symbol, ipo) -> str: + if not ipo: + return "No IPO microstructure data available." + lines = [ + f"IPO Microstructure — {symbol or 'N/A'}", + f"{'─'*46}", + f"IPO Symbol : {'YES' if ipo.is_ipo_symbol else 'No (standard analysis)'}", + f"Price Discovery Phase: {ipo.price_discovery_phase.replace('_', ' ').upper()}", + f"Stabilisation Likely : {'⚠ YES' if ipo.stabilisation_agent_likely else 'Not detected'}", + f"Greenshoe-like : {'⚠ YES' if ipo.greenshoe_activity_likely else 'Not detected'}", + f"Lock-up Proximity : {'⚠ SIGNAL ACTIVE' if ipo.lock_up_proximity_signal else 'Not detected'}", + f"Confidence : {ipo.confidence_label.upper()}", + "", + "IPO Signals:", + ] + for s in ipo.ipo_specific_signals: + lines.append(f" • {s}") + lines += [ + "", + "Note: Stabilisation and greenshoe labels describe observable patterns only.", + "Cross-reference with IPO prospectus (S-1) and lock-up expiry calendar.", + f"⚠️ {COMPLIANCE_LIMITATIONS_STATEMENT}", ] return "\n".join(lines) @@ -331,6 +550,16 @@ def _fmt_paper_trade(symbol, parl, risk) -> str: f" Risk Governor : {parl.risk_status.upper()}", f" Compliance : {parl.compliance_status.upper()}", ] + if parl.participant_archetype: + lines.append( + f" Archetype (Phase 2A): {parl.participant_archetype.replace('_', ' ')} " + f"({parl.participant_archetype_prob:.0%})" + ) + if parl.motivation_primary: + lines.append( + f" Motivation (Phase 2A): {parl.motivation_primary.replace('_', ' ')} " + f"[{parl.motivation_confidence}]" + ) if disp == "paper_trade_approved": lines.append( "\n✅ Setup approved for PAPER TRADE ONLY. " diff --git a/src/config.py b/src/config.py index 3fd6556..8afaaf5 100644 --- a/src/config.py +++ b/src/config.py @@ -26,6 +26,14 @@ OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "") ANTHROPIC_API_KEY: str = os.getenv("ANTHROPIC_API_KEY", "") +# ── DeepSeek (Phase 2A – Motivation Inference Engine) ───────────────────────── +# DeepSeek-R1 is accessed via an OpenAI-compatible API. +# Self-hosted local deployment is the preferred option for data sovereignty. +# Leave DEEPSEEK_API_KEY blank to use the rule-based motivation engine only. +DEEPSEEK_API_KEY: str = os.getenv("DEEPSEEK_API_KEY", "") +DEEPSEEK_BASE_URL: str = os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1") +DEEPSEEK_MODEL: str = os.getenv("DEEPSEEK_MODEL", "deepseek-reasoner") + # ── Storage ─────────────────────────────────────────────────────────────────── DATABASE_URL: str = os.getenv("DATABASE_URL", "sqlite:///data/microstructure.db") REDIS_URL: str = os.getenv("REDIS_URL", "redis://localhost:6379/0") @@ -65,3 +73,8 @@ "All behavioral classifications are probabilistic estimates based on " "observable, legally accessible market data only." ) + +# ── IPO configuration (Phase 2A) ───────────────────────────────────────────── +# Symbols in this list receive additional IPO microstructure analysis. +# Update with the active IPO ticker(s) you are monitoring. +IPO_SYMBOLS: list[str] = [s.strip().upper() for s in os.getenv("IPO_SYMBOLS", "SPCX").split(",") if s.strip()] diff --git a/src/orchestrator.py b/src/orchestrator.py index de9b31f..32d8023 100644 --- a/src/orchestrator.py +++ b/src/orchestrator.py @@ -23,9 +23,12 @@ from src.agents.compliance_agent import ComplianceAgent from src.agents.decision_parliament import DecisionParliament from src.agents.institutional_footprint import InstitutionalFootprintAgent +from src.agents.ipo_microstructure_agent import IPOMicrostructureAgent from src.agents.market_dna_detector import MarketDNADetectorAgent from src.agents.models import DecisionParliamentResult +from src.agents.motivation_inference_agent import MotivationInferenceAgent from src.agents.order_book_analyst import OrderBookAnalystAgent +from src.agents.participant_archetype_agent import ParticipantArchetypeAgent from src.agents.risk_governor import RiskGovernorAgent from src.audit.ledger import AuditLedger from src.chatbot.interface import ChatbotInterface, ChatbotResponse @@ -60,12 +63,20 @@ def __init__(self, symbol: str, feature_log_every: int = 10) -> None: self._parliament = DecisionParliament() self._chatbot = ChatbotInterface() self._ledger = AuditLedger() + # Phase 2A agents + self._archetype_agent = ParticipantArchetypeAgent() + self._motivation_agent = MotivationInferenceAgent() + self._ipo_agent = IPOMicrostructureAgent() # ── Latest state cache (for dashboard reads) ────────────────────────── self.latest_health: FeedHealthReport | None = None self.latest_state: OrderBookState | None = None self.latest_features: FeatureVector | None = None self.latest_result: DecisionParliamentResult | None = None + # Phase 2A caches + self.latest_archetype = None + self.latest_motivation = None + self.latest_ipo = None # ── Public ──────────────────────────────────────────────────────────────── @@ -106,13 +117,22 @@ def process( ) compliance_report = self._compliance.check_output(self.symbol, pre_text) - # 9. Decision parliament + # 9. Phase 2A — WHO / WHY / IPO agents + archetype_report = self._archetype_agent.analyse(state, features) + motivation_report = self._motivation_agent.analyse(state, features, archetype_report) + ipo_report = self._ipo_agent.analyse(state, features) + + # 10. Decision parliament (extended with Phase 2A reports) result = self._parliament.deliberate( - ob_report, dna_report, inst_report, risk_report, compliance_report + ob_report, dna_report, inst_report, risk_report, compliance_report, + archetype_report=archetype_report, + motivation_report=motivation_report, + ipo_report=ipo_report, ) - # 10. Audit + # 11. Audit self._ledger.log_decision(result) + self._ledger.log_motivation(motivation_report) if self._tick % self._feature_log_every == 0: self._ledger.log_features(features) @@ -121,6 +141,9 @@ def process( self.latest_state = state self.latest_features = features self.latest_result = result + self.latest_archetype = archetype_report + self.latest_motivation = motivation_report + self.latest_ipo = ipo_report return result @@ -149,6 +172,9 @@ def chat(self, user_input: str) -> ChatbotResponse: inst_report=inst_report, risk_report=risk_report, parliament_result=result, + archetype_report=self.latest_archetype, + motivation_report=self.latest_motivation, + ipo_report=self.latest_ipo, ) # Final compliance gate on the response text diff --git a/tests/test_agents.py b/tests/test_agents.py index 784553c..63076a7 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -8,6 +8,9 @@ from src.agents.order_book_analyst import OrderBookAnalystAgent from src.agents.market_dna_detector import MarketDNADetectorAgent from src.agents.institutional_footprint import InstitutionalFootprintAgent +from src.agents.participant_archetype_agent import ParticipantArchetypeAgent, ARCHETYPES +from src.agents.ipo_microstructure_agent import IPOMicrostructureAgent +from src.agents.motivation_inference_agent import MotivationInferenceAgent, MOTIVATION_TAXONOMY from src.agents.risk_governor import RiskGovernorAgent from src.agents.compliance_agent import ComplianceAgent from src.agents.decision_parliament import DecisionParliament @@ -253,3 +256,263 @@ def test_result_has_limitations(self): p = DecisionParliament() result = p.deliberate(ob, dna, inst, risk, comp) assert result.limitations != "" + + +# ── Phase 2A Tests ───────────────────────────────────────────────────────────── + +class TestParticipantArchetypeAgent: + def test_returns_report(self): + state, features, _ = _make_pipeline() + agent = ParticipantArchetypeAgent() + report = agent.analyse(state, features) + assert report.symbol == "TEST" + assert report.archetype in ARCHETYPES + assert 0 <= report.probability <= 1 + assert report.confidence_label in {"high", "medium", "low", "insufficient_data"} + + def test_limitations_always_populated(self): + state, features, _ = _make_pipeline() + agent = ParticipantArchetypeAgent() + report = agent.analyse(state, features) + assert report.limitations != "" + assert "level 2 data" in report.limitations.lower() + + def test_no_identity_in_archetype(self): + """Archetype label must never contain a named institution.""" + state, features, _ = _make_pipeline() + agent = ParticipantArchetypeAgent() + report = agent.analyse(state, features) + from src.config import IDENTITY_CLAIM_BLOCKED_TERMS + for term in IDENTITY_CLAIM_BLOCKED_TERMS: + assert term not in report.archetype.lower() + + def test_institutional_accumulator_on_strong_bid(self): + """Dominant bid side with large absorption should favour institutional_accumulator.""" + state, features, _ = _make_pipeline(bid_size=5000, ask_size=300) + agent = ParticipantArchetypeAgent() + report = agent.analyse(state, features) + # Should not classify as strategic_seller or retail_sentiment_buyer + assert report.archetype not in {"strategic_seller"} + + def test_secondary_archetype_optional(self): + state, features, _ = _make_pipeline() + agent = ParticipantArchetypeAgent() + report = agent.analyse(state, features) + # secondary_archetype may be None or a valid archetype + if report.secondary_archetype is not None: + assert report.secondary_archetype in ARCHETYPES + + +class TestIPOMicrostructureAgent: + def test_returns_report(self): + state, features, _ = _make_pipeline() + agent = IPOMicrostructureAgent() + report = agent.analyse(state, features) + assert report.symbol == "TEST" + assert report.price_discovery_phase in { + "pre_open", "price_discovery", "stabilisation", + "post_stabilisation", "normal_trading" + } + assert isinstance(report.ipo_specific_signals, list) + assert report.confidence_label in {"high", "medium", "low"} + + def test_spcx_is_ipo_symbol(self): + """SPCX must be recognised as an IPO symbol when configured.""" + from src.data_intake.models import L2Level, OrderBookSnapshot, Trade + engine = OrderBookEngine(depth_levels=5) + bids = [L2Level(price=round(100.0 - 0.05 - i * 0.01, 2), size=500) for i in range(5)] + asks = [L2Level(price=round(100.0 + 0.05 + i * 0.01, 2), size=400) for i in range(5)] + snap = OrderBookSnapshot(symbol="SPCX", bids=bids, asks=asks, feed="test") + trades = [Trade("SPCX", price=100.0, size=100, side="buy")] + state = engine.process(snap, trades) + eng = FeatureEngineer("SPCX") + features = eng.update(state, trades) + agent = IPOMicrostructureAgent() + report = agent.analyse(state, features) + assert report.is_ipo_symbol is True + + def test_greenshoe_requires_strong_conditions(self): + """Greenshoe flag should not fire on weak conditions.""" + state, features, _ = _make_pipeline(bid_size=300, ask_size=300) + agent = IPOMicrostructureAgent() + report = agent.analyse(state, features) + # With balanced and normal conditions, greenshoe should not be likely + # (not a guaranteed assertion — just confirm the report is valid) + assert isinstance(report.greenshoe_activity_likely, bool) + + +class TestMotivationInferenceAgent: + def _make_archetype_report(self, archetype="institutional_accumulator", prob=0.65, conf="high"): + from src.agents.models import ParticipantArchetypeReport + import time + return ParticipantArchetypeReport( + symbol="TEST", + timestamp_ns=time.time_ns(), + archetype=archetype, + probability=prob, + confidence_label=conf, + secondary_archetype=None, + secondary_probability=0.0, + evidence=["Test evidence signal."], + limitations="Level 2 data does not reveal participant identity.", + ) + + def test_returns_report(self): + state, features, _ = _make_pipeline() + arch = self._make_archetype_report() + agent = MotivationInferenceAgent() + report = agent.analyse(state, features, arch) + assert report.symbol == "TEST" + assert report.motivation_primary in MOTIVATION_TAXONOMY + assert report.motivation_confidence in {"high", "medium", "low"} + assert report.engine_used in {"rule_based", "deepseek"} + assert report.compliance_cleared is True + + def test_motivation_in_taxonomy(self): + state, features, _ = _make_pipeline() + for archetype in ["institutional_accumulator", "strategic_seller", + "algorithmic_market_maker", "momentum_ignitor", "unknown_mixed"]: + arch = self._make_archetype_report(archetype=archetype, prob=0.6) + agent = MotivationInferenceAgent() + report = agent.analyse(state, features, arch) + assert report.motivation_primary in MOTIVATION_TAXONOMY, ( + f"Motivation '{report.motivation_primary}' not in taxonomy for archetype '{archetype}'" + ) + + def test_alternative_motivation_in_taxonomy(self): + state, features, _ = _make_pipeline() + arch = self._make_archetype_report() + agent = MotivationInferenceAgent() + report = agent.analyse(state, features, arch) + assert report.motivation_alternative in MOTIVATION_TAXONOMY + + def test_reasoning_trace_populated(self): + state, features, _ = _make_pipeline() + arch = self._make_archetype_report() + agent = MotivationInferenceAgent() + report = agent.analyse(state, features, arch) + assert report.reasoning_trace != "" + + def test_rule_based_when_no_api_key(self): + """Without DEEPSEEK_API_KEY, engine must be rule_based.""" + import src.config as cfg + original = cfg.DEEPSEEK_API_KEY + cfg.DEEPSEEK_API_KEY = "" + try: + state, features, _ = _make_pipeline() + arch = self._make_archetype_report() + agent = MotivationInferenceAgent() + report = agent.analyse(state, features, arch) + assert report.engine_used == "rule_based" + finally: + cfg.DEEPSEEK_API_KEY = original + + +class TestDecisionParliamentPhase2A: + """Tests that Phase 2A extended fields appear correctly in parliament results.""" + + def _make_full_reports(self): + from src.agents.models import ( + OrderBookAnalystReport, MarketDNAReport, InstitutionalFootprintReport, + RiskGovernorReport, ComplianceReport, ParticipantArchetypeReport, MotivationInferenceReport, + ) + import time + ts = time.time_ns() + ob = OrderBookAnalystReport( + symbol="TEST", timestamp_ns=ts, + directional_bias="bullish", bias_confidence=0.7, + key_signals=[], absorption_score=0.5, sweep_intensity=0.2, + spoof_like_score=0.1, iceberg_like_score=0.3, + spread_multiple=1.0, book_imbalance=0.2, + ) + dna = MarketDNAReport( + symbol="TEST", timestamp_ns=ts, + regime="accumulation", regime_confidence=0.65, + supporting_signals=[], regime_stability=0.7, + ) + inst = InstitutionalFootprintReport( + symbol="TEST", timestamp_ns=ts, + behavioral_label="accumulation_like", probability=0.68, + confidence_label="high", reasoning=[], limitations="test", + ) + risk = RiskGovernorReport( + symbol="TEST", timestamp_ns=ts, + risk_status="clear", risk_score=0.1, + veto=False, veto_reasons=[], + spread_ok=True, data_quality_ok=True, + model_confidence_ok=True, news_lockout=False, + ) + comp = ComplianceReport( + symbol="TEST", timestamp_ns=ts, + status="approved", violations=[], warnings=[], + ) + arch = ParticipantArchetypeReport( + symbol="TEST", timestamp_ns=ts, + archetype="institutional_accumulator", probability=0.70, + confidence_label="high", secondary_archetype=None, + secondary_probability=0.0, evidence=["Test evidence."], + limitations="test", + ) + motiv = MotivationInferenceReport( + symbol="TEST", timestamp_ns=ts, + motivation_primary="position_building_ahead_of_catalyst", + motivation_confidence="high", + motivation_evidence=["Test evidence."], + motivation_alternative="mandate_driven_allocation", + archetype_context="institutional_accumulator", + reasoning_trace="Test trace.", + engine_used="rule_based", + compliance_cleared=True, + ) + return ob, dna, inst, risk, comp, arch, motiv + + def test_phase2a_fields_in_result(self): + ob, dna, inst, risk, comp, arch, motiv = self._make_full_reports() + p = DecisionParliament() + result = p.deliberate(ob, dna, inst, risk, comp, + archetype_report=arch, motivation_report=motiv) + assert result.participant_archetype == "institutional_accumulator" + assert result.motivation_primary == "position_building_ahead_of_catalyst" + assert result.motivation_confidence == "high" + + def test_phase2a_fields_none_when_not_provided(self): + """Backward compatibility: Phase 2A fields are None when reports not passed.""" + from src.agents.models import ( + OrderBookAnalystReport, MarketDNAReport, InstitutionalFootprintReport, + RiskGovernorReport, ComplianceReport, + ) + import time + ts = time.time_ns() + ob = OrderBookAnalystReport( + symbol="TEST", timestamp_ns=ts, + directional_bias="neutral", bias_confidence=0.4, + key_signals=[], absorption_score=0.2, sweep_intensity=0.1, + spoof_like_score=0.1, iceberg_like_score=0.1, + spread_multiple=1.0, book_imbalance=0.1, + ) + dna = MarketDNAReport( + symbol="TEST", timestamp_ns=ts, + regime="ranging", regime_confidence=0.5, + supporting_signals=[], regime_stability=0.7, + ) + inst = InstitutionalFootprintReport( + symbol="TEST", timestamp_ns=ts, + behavioral_label="neutral", probability=0.3, + confidence_label="low", reasoning=[], limitations="test", + ) + risk = RiskGovernorReport( + symbol="TEST", timestamp_ns=ts, + risk_status="clear", risk_score=0.1, + veto=False, veto_reasons=[], + spread_ok=True, data_quality_ok=True, + model_confidence_ok=True, news_lockout=False, + ) + comp = ComplianceReport( + symbol="TEST", timestamp_ns=ts, + status="approved", violations=[], warnings=[], + ) + p = DecisionParliament() + result = p.deliberate(ob, dna, inst, risk, comp) + assert result.participant_archetype is None + assert result.motivation_primary is None + assert result.ipo_phase is None From 55728635894908e26af60aedea2a8e254953c21b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 05:36:38 +0000 Subject: [PATCH 4/4] fix: use monkeypatch for test isolation; improve DeepSeek import error message --- src/agents/motivation_inference_agent.py | 9 +++++++-- tests/test_agents.py | 18 +++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/agents/motivation_inference_agent.py b/src/agents/motivation_inference_agent.py index ea41f5e..2485794 100644 --- a/src/agents/motivation_inference_agent.py +++ b/src/agents/motivation_inference_agent.py @@ -116,7 +116,6 @@ def analyse( engine = "deepseek" except Exception: pass # Silently fall back to rule-based - return MotivationInferenceReport( symbol=state.symbol, timestamp_ns=state.timestamp_ns, @@ -228,7 +227,13 @@ def _deepseek_enrich( and return the reasoning trace. Never replaces compliance-cleared rule-based output — only enriches the reasoning narrative. """ - import openai # imported lazily to avoid hard dependency when not needed + try: + import openai # imported lazily — requires 'openai' package when DEEPSEEK_API_KEY is set + except ImportError as exc: + raise ImportError( + "The 'openai' package is required for DeepSeek integration. " + "Install it with: pip install openai>=1.12.0" + ) from exc client = openai.OpenAI( api_key=DEEPSEEK_API_KEY, diff --git a/tests/test_agents.py b/tests/test_agents.py index 63076a7..f53dfb7 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -393,19 +393,15 @@ def test_reasoning_trace_populated(self): report = agent.analyse(state, features, arch) assert report.reasoning_trace != "" - def test_rule_based_when_no_api_key(self): + def test_rule_based_when_no_api_key(self, monkeypatch): """Without DEEPSEEK_API_KEY, engine must be rule_based.""" import src.config as cfg - original = cfg.DEEPSEEK_API_KEY - cfg.DEEPSEEK_API_KEY = "" - try: - state, features, _ = _make_pipeline() - arch = self._make_archetype_report() - agent = MotivationInferenceAgent() - report = agent.analyse(state, features, arch) - assert report.engine_used == "rule_based" - finally: - cfg.DEEPSEEK_API_KEY = original + monkeypatch.setattr(cfg, "DEEPSEEK_API_KEY", "") + state, features, _ = _make_pipeline() + arch = self._make_archetype_report() + agent = MotivationInferenceAgent() + report = agent.analyse(state, features, arch) + assert report.engine_used == "rule_based" class TestDecisionParliamentPhase2A: