diff --git a/.env.template b/.env.template index d08efcff..eb407501 100644 --- a/.env.template +++ b/.env.template @@ -56,7 +56,7 @@ SANDBOX=local # code execution backend: 'local' (default) or 'docke # OpenAI OPENAI_ENABLED=true OPENAI_API_KEY=#your-openai-api-key -OPENAI_MODELS=gpt-5.4,gpt-4.1 # comma separated list of models +OPENAI_MODELS=gpt-5.5 # comma separated list of models # Azure OpenAI AZURE_ENABLED=true diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5c35cc99..d770c860 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,24 @@ updates: target-branch: "dev" schedule: interval: weekly + + # Frontend JavaScript dependencies (package.json / yarn.lock) + - package-ecosystem: "npm" + directory: "/" + target-branch: "dev" + schedule: + interval: weekly + + # Backend Python dependencies (pyproject.toml / uv.lock) + - package-ecosystem: "uv" + directory: "/" + target-branch: "dev" + schedule: + interval: weekly + + # GitHub Actions workflow dependencies + - package-ecosystem: "github-actions" + directory: "/" + target-branch: "dev" + schedule: + interval: weekly diff --git a/README.md b/README.md index 78b81fd1..28d40020 100644 --- a/README.md +++ b/README.md @@ -43,14 +43,14 @@ https://github.com/user-attachments/assets/8e4f8a08-6423-4227-a1f7-559e0126ce31 ## News 🔥🔥🔥 -[07-13-2026] **Data Formulator 0.8 alpha 1** — a preview of a more connected, agent-driven data workflow: +[07-17-2026] **Data Formulator 0.8 alpha 2** — A more connected, conversational way to work with data: -- **Smarter agent handoffs.** Analysis can delegate directly to data loading while preserving the conversation and request context. -- **A redesigned connector experience.** Progressive setup, explicit authentication paths, database discovery, clearer scope controls, and improved credential handling make data sources easier to configure safely. -- **Better loading plans.** Recommended selections, compact previews, readable filters, provenance, and more reliable history and scrolling improve review before import. -- **Stronger enterprise foundations.** Unified connector metadata, session-scoped knowledge and distillation improvements, model routing, and additional isolation guardrails prepare Data Formulator for larger deployments. +- **Explore large datasets through conversation.** Connect a database, then ask the agent to find the right tables, filter the data, or update your selection as your question evolves. You can review the resulting filters, previews, and data sources before anything is loaded. +- **Keep the whole analysis in one conversation.** Agents can load data without losing track of what you asked. Questions, explanations, and results stay together in the Data Thread, so you can pick up from any step or branch into a new question, column, or chart. +- **Choose a chart that fits the question.** The gallery now includes bullet, connected scatter, ECDF, Gantt, range area, slope, sparkline, and violin charts, along with better recommendations. Files you attach also stay available to the analyst as the exploration continues. +- **Spend less time troubleshooting.** This release improves long-running sessions, model routing, data isolation, installation across platforms, dependency security, and MySQL data freshness. Persistent logs and an in-app log viewer make problems easier to track down. -> Preview with `pip install --pre data_formulator==0.8.0a1` or `uvx --from data_formulator==0.8.0a1 data_formulator`. +> Preview with `pip install --pre data_formulator==0.8.0a2` or `uvx --from data_formulator==0.8.0a2 data_formulator`. > Install the latest stable release (0.7) with `pip install data_formulator` or run instantly with `uvx data_formulator`. diff --git a/package.json b/package.json index 5ce26909..5ff8f332 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,10 @@ "resolutions": { "lodash": "^4.18.1", "vite": "^7.3.3", - "dompurify": "^3.4.2" + "dompurify": "^3.4.2", + "markdown-it": "^14.3.0", + "linkify-it": "^5.0.2", + "undici": "^7.28.0" }, "dependencies": { "@azure/msal-browser": "^5.6.3", @@ -60,13 +63,14 @@ "react-katex": "^3.1.0", "react-markdown": "^10.1.0", "react-redux": "^8.0.4", - "react-router-dom": "^6.22.0", + "react-router-dom": "^6.30.4", "react-selectable-fast": "^3.4.0", "react-simple-code-editor": "^0.13.1", "react-vega": "^7.6.0", "react-virtuoso": "^4.3.10", "redux": "^4.2.0", "redux-persist": "^6.0.0", + "remark-gfm": "^4", "tiptap-markdown": "^0.9.0", "typescript": "^4.9.5", "validator": "^13.15.20", diff --git a/py-src/data_formulator/agents/agent_data_loading_chat.py b/py-src/data_formulator/agents/agent_data_loading_chat.py index 7f5d36d4..e31fda62 100644 --- a/py-src/data_formulator/agents/agent_data_loading_chat.py +++ b/py-src/data_formulator/agents/agent_data_loading_chat.py @@ -1268,9 +1268,14 @@ def _tool_execute_python(self, args): return response else: + err = raw.get("error_message", raw.get("content", "Unknown error")) + logger.warning( + "execute_python code failed: %s\n--- code ---\n%s", + err, code[:2000], + ) return { "stdout": "", - "error": raw.get("error_message", raw.get("content", "Unknown error")), + "error": err, } except Exception as e: diff --git a/py-src/data_formulator/agents/client_utils.py b/py-src/data_formulator/agents/client_utils.py index efd99d51..b222e507 100644 --- a/py-src/data_formulator/agents/client_utils.py +++ b/py-src/data_formulator/agents/client_utils.py @@ -345,7 +345,7 @@ def ping(self, timeout: int = 10): params["timeout"] = timeout litellm.completion( model=self.model, messages=messages, - max_tokens=3, drop_params=True, **params, + max_tokens=3, drop_params=True, _skip_mcp_handler=True, **params, ) def _dispatch(self, *, messages, stream, params, tools=None, extra=None): @@ -359,6 +359,11 @@ def _dispatch(self, *, messages, stream, params, tools=None, extra=None): effective_stream = stream and not is_ollama call_kwargs = dict(model=self.model, messages=messages, drop_params=True, stream=effective_stream, + # We never use litellm's built-in MCP gateway. Setting this + # skips litellm's proxy/MCP handler import path, which pulls + # in fastapi and is not a dependency of this project + # (litellm>=1.92 imports it whenever `tools` are passed). + _skip_mcp_handler=True, **params, **(extra or {})) if tools is not None: call_kwargs["tools"] = tools diff --git a/py-src/data_formulator/agents/semantic_types.py b/py-src/data_formulator/agents/semantic_types.py index 7731dd8f..be6e9a80 100644 --- a/py-src/data_formulator/agents/semantic_types.py +++ b/py-src/data_formulator/agents/semantic_types.py @@ -6,8 +6,9 @@ SEMANTIC TYPE SYSTEM (Python mirror of the TypeScript registry) ============================================================================= -The **source of truth** for semantic types lives in the TypeScript library: - src/lib/agents-chart/core/type-registry.ts +The **source of truth** for semantic types lives in the flint-chart library +(npm package `flint-chart`, repo microsoft/flint-chart): + packages/flint-js/src/core/type-registry.ts This file mirrors the registered types and provides: 1. String constants for every type in the TS TYPE_REGISTRY diff --git a/py-src/data_formulator/analyst/agent.py b/py-src/data_formulator/analyst/agent.py index 078fb610..df3bf60f 100644 --- a/py-src/data_formulator/analyst/agent.py +++ b/py-src/data_formulator/analyst/agent.py @@ -215,14 +215,14 @@ def _decode(self, args: str) -> str | None: next message starts a fresh turn). Use it whenever you've done what was asked, including answering a question you fully resolved. -**Whenever you expect the user to reply — a question, a clarification, an -explanation you want them to react to, or a set of choices — use the `ask_user` -action instead.** It renders a question widget and pauses the run for their -reply, so the conversation resumes in the same turn. `ask_user` accepts -free-text questions (no clickable options required), so reach for it for *any* -followup-seeking turn, not only structured choices. Plain text never asks for -input; `ask_user` always does. There is no separate "stop" or "summary" action: -you stop by simply not acting. +**Whenever you expect the user to reply — a question, a clarification, or a set +of choices — use the `ask_user` action instead.** It renders a question widget +and pauses the run for their reply, so the conversation resumes in the same +turn. `ask_user` accepts free-text questions (no clickable options required), so +reach for it for *any* followup-seeking turn, not only structured choices. Keep +your reasoning and explanations in your reply text, not inside `ask_user`. Plain +text never asks for input; `ask_user` always does. There is no separate "stop" +or "summary" action: you stop by simply not acting. The concrete actions available to you — and how to use each well — are described in the capability sections below. @@ -371,6 +371,7 @@ def run( primary_tables: list[str] | None = None, attached_images: list[str] | None = None, charts: list[dict[str, Any]] | None = None, + scratch_files: list[str] | None = None, ) -> Generator[dict[str, Any], None, None]: """Run the unified analyst loop. @@ -434,6 +435,7 @@ def run( primary_tables=primary_tables, attached_images=attached_images, charts=charts, + scratch_files=scratch_files, ) rlog.log( "context_built", @@ -973,9 +975,14 @@ def _run_explore_code( stdout = stdout[:8000] + "\n... (truncated)" return {"status": "ok", "stdout": stdout} else: + err = raw.get("error_message", raw.get("content", "Unknown error")) + logger.warning( + "[AnalystAgent] explore code failed: %s\n--- code ---\n%s", + err, code[:2000], + ) return { "status": "error", - "error": raw.get("error_message", raw.get("content", "Unknown error")), + "error": err, "stdout": "", } except Exception as e: @@ -1019,6 +1026,10 @@ def _run_visualize_code( if execution_result['status'] != 'ok': error_message = execution_result.get('content', 'Unknown error') + logger.warning( + "[AnalystAgent] visualize code failed: %s\n--- code ---\n%s", + error_message, code[:2000], + ) return {"status": "error", "error_message": str(error_message)} full_df = execution_result['content'] @@ -1237,6 +1248,7 @@ def _build_initial_messages( primary_tables: list[str] | None = None, attached_images: list[str] | None = None, charts: list[dict[str, Any]] | None = None, + scratch_files: list[str] | None = None, ) -> list[dict]: """Build the initial messages with 3-tier context.""" table_summaries = self._build_lightweight_table_context(input_tables, primary_tables=primary_tables) @@ -1273,6 +1285,25 @@ def _build_initial_messages( rules_text = "\n\n".join([f"### {r['title']}\n{r['body']}" for r in always_apply_rules]) user_content += f"[USER RULES - MUST FOLLOW]\n\n{rules_text}\n\n" + # Non-image attachments were uploaded to the workspace scratch/ folder + # (raw bytes). Surface them and the two natural uses: read as context + # for analysis, or bring into the workspace via data loading (which + # shares scratch/ access). Keep it neutral — the agent picks. + if scratch_files: + file_list = "\n".join(f"- {p}" for p in scratch_files) + user_content += ( + "[ATTACHED FILES]\n\n" + "The user attached file(s), saved in the workspace scratch/ " + "folder:\n" + f"{file_list}\n\n" + "You can use them two ways: read a file with execute_python_script " + "(e.g. pd.read_excel('scratch/') or " + "pd.read_csv('scratch/')) to use as context for your " + "analysis, or \u2014 if it's data to bring into the workspace \u2014 " + "delegate to data_loading, which can read the same scratch/ " + "files.\n\n" + ) + user_content += f"[USER QUESTION]\n\n{user_question}" system_prompt = self._build_system_prompt( diff --git a/py-src/data_formulator/analyst/mini_agent.py b/py-src/data_formulator/analyst/mini_agent.py index 99088c3c..80b5ea49 100644 --- a/py-src/data_formulator/analyst/mini_agent.py +++ b/py-src/data_formulator/analyst/mini_agent.py @@ -442,6 +442,7 @@ def run( primary_tables: list[str] | None = None, attached_images: list[str] | None = None, charts: list[dict[str, Any]] | None = None, + scratch_files: list[str] | None = None, ) -> Generator[dict[str, Any], None, None]: """Make a single analytic decision and stop. @@ -486,6 +487,7 @@ def run( input_tables, user_question, focused_thread, other_threads, primary_tables=primary_tables, attached_images=attached_images, charts=charts, + scratch_files=scratch_files, ) else: messages = trajectory diff --git a/py-src/data_formulator/analyst/skills/core/SKILL.md b/py-src/data_formulator/analyst/skills/core/SKILL.md index de31afa2..7303c395 100644 --- a/py-src/data_formulator/analyst/skills/core/SKILL.md +++ b/py-src/data_formulator/analyst/skills/core/SKILL.md @@ -91,22 +91,24 @@ result and decide your next move. ### `ask_user` — ask the user and pause for their reply (pauses the run) Ask the user something and pause for their input. Reach for this on **any** turn -where you want a reply — a freeform question, a clarification you need before -acting, or an explanation you want them to react to. Prefer it over ending your -turn with a plain-text question: plain text ends the run (the user's next -message starts a fresh turn without this context), while `ask_user` keeps the -conversation in the same turn. - -- `questions` — 1–3 items. Each is either a question that awaits an answer - (clarification) or a statement the user need not answer (explanation). A - question with no required answer and no options renders as a plain - explanation; offer chart-producing follow-ups as its `options`. +where you want a reply — a choice to make, a clarification you need before +acting, or a brief statement paired with clickable follow-ups they can react to. +Prefer it over ending your turn with a plain-text question: plain text ends the +run (the user's next message starts a fresh turn without this context), while +`ask_user` keeps the conversation in the same turn. + +- `questions` — 1–3 items, each something the user **acts on**: a choice + (`single_choice` with `options`) or an open question they type an answer to + (`free_text`). Put your reasoning, rationale, and context in your reply text — + **not** here. Never add a `questions` item that only states a rationale or + explanation with nothing for the user to answer or click. - each question: `text` (wrap a **column** in `**…**`), `responseType` - (`single_choice` when offering `options`, else `free_text`), `required` - (`true` for a clarification the run depends on, `false` for an explanation / - optional follow-up), and `options` (plain-text choices, **at most 3** — just - the most likely answers; the user can always type a freeform reply, so don't - enumerate every case). + (`single_choice` when you offer `options`, else `free_text` — the user types + their own open-ended answer, not a slot for your exposition), `required` + (`true` when the run depends on the answer, `false` for an optional follow-up), + and `options` (plain-text choices, **at most 3** — just the most likely + answers; the user can always type a freeform reply, so don't enumerate every + case). This is **terminal**: the run pauses after it and resumes when the user replies. @@ -204,28 +206,53 @@ The following reference material applies when you call the `visualize` tool. ### B. Chart Type Reference The `chart_type` value in the `visualize` action MUST be one of the names listed -in the first column below (exact spelling, including capitalization). When a row -lists multiple names, pick whichever fits the "when to use" hint best. +below (exact spelling, including capitalization). When a row lists multiple +names, pick whichever fits the "when to use" hint best. + +**Choosing a chart — prefer simple, escalate when it fits.** Reach for the +**Everyday** set first: it answers most questions and is the safest, most +legible choice. But when the data or question genuinely fits a **Specialized** +type (a distribution's shape, a cumulative curve, a rank race, a before→after, +a geographic pattern…), prefer it — a well-matched specialized chart is more +insightful than forcing a generic one. Don't pick a specialized type for +novelty; use it because its "when to use" condition is met. + +**Everyday — reach for these first** | chart_type | encodings | config | when to use | |---|---|---|---| | Scatter Plot | x, y, color, size, facet | opacity (0.1–1.0) | Relationships between two quantitative fields | | Regression | x, y, color, size, facet | regressionMethod ("linear","log","exp","pow","quad","poly"), polyOrder (2–10) | Trend line over scatter; one line per color group | -| Bar Chart / Lollipop Chart / Waterfall Chart | x, y, color, facet | — | Bar: default categorical comparison. Lollipop: cleaner for ranked lists / sparse categories. Waterfall: cumulative gain/loss, each bar starts where the previous ended | +| Bar Chart / Stacked Bar Chart / Lollipop Chart / Waterfall Chart | x, y, color, facet | — | Bar: categorical comparison (auto-stacks when color is set). Stacked Bar: explicit stacked totals, color = the stack. Lollipop: cleaner for ranked lists / sparse categories. Waterfall: cumulative gain/loss, each bar starts where the previous ended | | Grouped Bar Chart | x, y, group, facet | — | Side-by-side bars across a second categorical dimension | -| Histogram / Density Plot | x, color, facet | — | Distribution of one quantitative field. Histogram: discrete bins, auto-binned. Density Plot: smooth KDE curve | -| Boxplot | x, y, color, facet | — | Distribution summary (median/quartiles/outliers) by category | -| Ranged Dot Plot | x, y, color, facet | — | Min–max range or two-point comparison per category | | Line Chart | x, y, color, strokeDash, facet | interpolate ("linear","monotone","step") | Trends over an ordered (usually temporal) x-axis | | Area Chart | x, y, color, facet | — | Magnitude over ordered x; auto-stacks when color is set | +| Histogram / Density Plot | x, color, facet | — | Distribution of one quantitative field. Histogram: discrete bins, auto-binned. Density Plot: smooth KDE curve | +| Boxplot | x, y, color, facet | — | Distribution summary (median/quartiles/outliers) by category | | Pie Chart | size, color, facet | innerRadius (0–100; 0=pie, >0=donut) | Part-of-whole with ≤7 categories. Wedge value goes on **size**, not **theta** | -| Radar Chart | x, y, color, facet | — | Multi-metric profile/comparison; x = metric name, y = value, color = entity (long-form data) | | Heatmap | x, y, color, facet | colorScheme — sequential ("viridis","blues","reds","oranges","greens") or diverging ("blueorange","redblue") | Matrix / 2D density; color encodes the quantitative cell value | + +**Specialized — use when the data/question fits the "when to use"** + +| chart_type | encodings | config | when to use | +|---|---|---|---| +| Connected Scatter Plot | x, y, order, color, facet | — | Two quantitative fields traced in sequence — needs an `order` field (e.g. time) so points are joined in order, not by x | +| Ranged Dot Plot | x, y, color, facet | — | Min–max range or two-point comparison per category | +| Violin Plot | x, y, color, facet | — | Distribution SHAPE (KDE silhouette) by category; better than a boxplot when data is multimodal. x = category, y = value | +| Strip Plot | x, y, color, size, facet | — | Every individual point by category (jittered); good for small/medium n where raw values matter, not just a summary | +| ECDF Plot | x, color, facet | — | Cumulative distribution of one quantitative field. Pass the RAW field on x (do NOT pre-compute the CDF); color for per-group curves | +| Bump Chart | x, y, color, facet | — | How RANKINGS change over ordered x; y = rank, color = entity (long-form: one row per entity × x) | +| Slope Chart | x, y, color, facet | — | Change between exactly TWO points (before → after) per entity; x = the two labels, y = value, color = entity | +| Streamgraph | x, y, color, facet | — | Several series' magnitude over ordered x, stacked around a center baseline (color = series) — theme/volume shifts over time | +| Range Area Chart | x, y, y2, color, facet | — | A shaded band between a lower (y) and upper (y2) bound over ordered x — e.g. min–max or a confidence interval | +| Rose Chart | x, y, color, facet | — | Cyclical/categorical magnitude as angular wedges (polar bars); x = category/angle, y = value | +| Pyramid Chart | x, y, color, facet | — | Back-to-back bars split by a binary group (e.g. population by age × sex); y = category, x = value, color = the two-sided group | +| Radar Chart | x, y, color, facet | — | Multi-metric profile/comparison; x = metric name, y = value, color = entity (long-form data) | | Bar Table | x, y, color, facet | — | Ranked horizontal table with inline bars; one row per category. y = category, x = value | | KPI Card | metric, value, goal | — | "Big number" dashboard tile(s); one row per tile. `value` must be pre-aggregated; `goal` is optional | | Candlestick Chart | x, open, high, low, close, facet | — | OHLC financial data | -| World Map | longitude, latitude, color, size | projection ("mercator","equalEarth","naturalEarth1","orthographic"), projectionCenter ([lon,lat]) | Geographic points/regions on a world projection | -| US Map | longitude, latitude, color, size | — (fixed albersUsa) | US-only points/regions (albersUsa projection) | +| Map | longitude, latitude, color, size | projection ("mercator","equalEarth","naturalEarth1","orthographic","albersUsa"), projectionCenter ([lon,lat]) | Geographic POINTS/bubbles by lon/lat (use projection "albersUsa" for a US-only map) | +| Choropleth | id, color, facet | region ("world","usa",…) | Filled REGIONS shaded by value; `id` = the region key (country/state name or code), color = the quantitative value | **Critical chart rules:** - **Scatter Plot**: use config opacity (0.1–1.0) for dense data instead of encoding opacity. @@ -240,7 +267,12 @@ lists multiple names, pick whichever fits the "when to use" hint best. - **Bar Table**: y is the category column to rank; x is the quantitative value driving bar length. Don't sort in Python — the template sorts. - **KPI Card**: channels are `metric`, `value`, `goal` (not x/y). One DataFrame row = one tile. The `value` column must already contain the final number to display (aggregate upstream in the Python step). - **Candlestick Chart**: requires `open`, `high`, `low`, `close` columns. -- **World Map / US Map**: channel names are `longitude` / `latitude`, not `x` / `y`. +- **Connected Scatter Plot**: provide an `order` field (usually time) so points are joined in sequence, not by x-order. +- **ECDF Plot**: pass the RAW quantitative field on `x` — the chart computes the cumulative curve; do NOT pre-compute it in Python. +- **Range Area Chart**: `y` is the lower bound and `y2` the upper bound of the band. +- **Bump / Slope Chart**: long-form data — one row per (entity, x); `color` is the entity. Slope's `x` has exactly two categories (before/after). +- **Violin Plot**: like Boxplot but shows the full distribution shape; x = category, y = value. +- **Map / Choropleth**: `Map` plots points via `longitude` / `latitude` (set projection `"albersUsa"` for the US); `Choropleth` fills regions — put the region key on `id` and the value on `color`, not `x` / `y`. - **facet**: available for nearly all chart types; use a low-cardinality categorical field. - All fields in `encodings` must also appear in `output_fields`. Typically use 2–3 channels (x, y, color/size). diff --git a/py-src/data_formulator/analyst/skills/core/tools.json b/py-src/data_formulator/analyst/skills/core/tools.json index a433988a..76b6b1a7 100644 --- a/py-src/data_formulator/analyst/skills/core/tools.json +++ b/py-src/data_formulator/analyst/skills/core/tools.json @@ -88,7 +88,7 @@ "type": "function", "function": { "name": "ask_user", - "description": "Ask the user something and pause for their reply — the run resumes in the same turn with their answer in context. Use this for ANY turn where you want the user to respond: a freeform question, a clarification you need before acting, or an explanation you want them to react to. Freeform is fine (no clickable options required). Prefer this over ending your turn with a plain-text question: plain text ends the run and the user's next message starts a fresh turn without this context, whereas ask_user keeps the conversation going. Reserve plain text (no action) for your final answer when you expect nothing further.", + "description": "Ask the user something and pause for their reply — the run resumes in the same turn with their answer in context. Use this for ANY turn where you want the user to respond: a choice to make, a clarification you need before acting, or a brief statement paired with clickable follow-ups. Put your reasoning, rationale, and context in your normal reply text, not inside this call. Prefer this over ending your turn with a plain-text question: plain text ends the run and the user's next message starts a fresh turn without this context, whereas ask_user keeps the conversation going. Reserve plain text (no action) for your final answer when you expect nothing further.", "parameters": { "type": "object", "properties": { @@ -98,18 +98,18 @@ }, "questions": { "type": "array", - "description": "One or more things to put to the user. Each is either a question that awaits an answer (clarification) or a statement the user need not answer (explanation). A question with no required answer and no options renders as a plain explanation. Ask at most 3.", + "description": "1–3 things the user acts on: a choice (single_choice with options) or an open question they type an answer to (free_text). Put rationale, reasoning, and context in your reply text, not here — never add an item that only states an explanation with nothing for the user to answer or click. An explanation is allowed only as a short statement paired with clickable chart-producing follow-ups (required=false with options).", "items": { "type": "object", "properties": { "text": { "type": "string", - "description": "The question or explanation. For an explanation, keep it to 1–3 grounded sentences. Wrap a **column** in ** ** to highlight it." + "description": "The question, or (for an optional follow-up) a short statement. Keep a statement to 1–3 grounded sentences and pair it with clickable follow-up options. Wrap a **column** in ** ** to highlight it." }, "responseType": { "type": "string", "enum": ["single_choice", "free_text"], - "description": "single_choice when options are offered; free_text when the user types a custom answer." + "description": "single_choice when you offer options; free_text when the user types their own open-ended answer (not a slot for your own exposition)." }, "required": { "type": "boolean", diff --git a/py-src/data_formulator/app.py b/py-src/data_formulator/app.py index 64bf5158..dde1a5c0 100644 --- a/py-src/data_formulator/app.py +++ b/py-src/data_formulator/app.py @@ -123,6 +123,85 @@ def default(self, obj): # Get logger for this module (logging config moved to run_app function) logger = logging.getLogger(__name__) +_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +# Marker attribute so we never attach the persistent file handler twice +# (configure_logging runs early; run_app re-checks once --data-dir is known). +_FILE_HANDLER_MARKER = '_df_persistent_file_handler' + + +def _resolve_data_home() -> Path: + """Resolve the Data Formulator home directory for logging. + + Mirrors ``get_data_formulator_home()`` but is safe to call outside an + application context (logging is configured before the app runs). + Order: CLI_ARGS['data_dir'] > DATA_FORMULATOR_HOME env > ~/.data_formulator. + """ + data_dir = app.config.get('CLI_ARGS', {}).get('data_dir') + if data_dir: + return Path(data_dir) + env_home = os.getenv('DATA_FORMULATOR_HOME') + if env_home: + return Path(env_home) + return Path.home() / '.data_formulator' + + +def configure_file_logging() -> None: + """Attach a rotating file handler that persists all logs under + ``/logs/data_formulator.log``. + + This is the artifact users can send when reporting problems. Output is + passed through :class:`SensitiveDataFilter` so API keys / tokens are + redacted. The handler is idempotent — if the resolved path changes + (e.g. ``--data-dir`` supplied after early configuration) the old handler + is replaced. + """ + from logging.handlers import RotatingFileHandler + from data_formulator.security.log_sanitizer import SensitiveDataFilter + + try: + log_dir = _resolve_data_home() / 'logs' + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / 'data_formulator.log' + except Exception as exc: # pragma: no cover - filesystem edge cases + logging.getLogger('data_formulator').warning( + "Could not set up persistent file logging: %s", exc, + ) + return + + root = logging.getLogger() + + # Skip if an identical handler is already attached; otherwise drop any + # previous persistent handler so we don't duplicate output. + for h in list(root.handlers): + if getattr(h, _FILE_HANDLER_MARKER, False): + if getattr(h, 'baseFilename', None) == str(log_path.resolve()): + app.config['LOG_FILE_PATH'] = str(log_path) + return + root.removeHandler(h) + try: + h.close() + except Exception: + pass + + handler = RotatingFileHandler( + log_path, maxBytes=10 * 1024 * 1024, backupCount=5, encoding='utf-8', + ) + setattr(handler, _FILE_HANDLER_MARKER, True) + handler.setLevel(logging.DEBUG) + handler.setFormatter(logging.Formatter(_LOG_FORMAT)) + handler.addFilter(SensitiveDataFilter()) + root.addHandler(handler) + + # Mirror onto the Flask app logger too (its handlers are managed separately). + if not any(getattr(h, _FILE_HANDLER_MARKER, False) for h in app.logger.handlers): + app.logger.addHandler(handler) + + app.config['LOG_FILE_PATH'] = str(log_path) + logging.getLogger('data_formulator').info( + "Persistent logs: %s", log_path, + ) + + def configure_logging(): """Configure logging for the Flask application.""" log_level_str = os.getenv("LOG_LEVEL", "INFO").strip().upper() @@ -135,7 +214,7 @@ def configure_logging(): logging.basicConfig( level=logging.WARNING, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + format=_LOG_FORMAT, handlers=[handler], ) @@ -149,6 +228,9 @@ def configure_logging(): for h in logging.getLogger().handlers: app.logger.addHandler(h) + # Persist everything to a file under the DF home so users can send logs. + configure_file_logging() + logging.getLogger('data_formulator').info( "Log level: %s (sanitize=%s)", log_level_str, os.getenv("LOG_SANITIZE", "true"), @@ -188,12 +270,16 @@ def _register_blueprints(): # Import demo stream routes from data_formulator.routes.demo_stream import demo_stream_bp, limiter as demo_stream_limiter demo_stream_limiter.init_app(app) - + + # Import server-log inspection routes (local-mode gated) + from data_formulator.routes.logs import logs_bp + # Register blueprints app.register_blueprint(tables_bp) app.register_blueprint(agent_bp) app.register_blueprint(session_bp) app.register_blueprint(demo_stream_bp) + app.register_blueprint(logs_bp) # Initialise pluggable authentication (reads AUTH_PROVIDER env var) from data_formulator.auth.identity import init_auth, get_active_provider @@ -446,6 +532,10 @@ def run_app(): ], } + # Now that --data-dir is applied, ensure the persistent log file lives + # under the resolved DF home (re-points if it differs from the default). + configure_file_logging() + # Register blueprints (this is where heavy imports happen) _register_blueprints() diff --git a/py-src/data_formulator/data_loader/mysql_data_loader.py b/py-src/data_formulator/data_loader/mysql_data_loader.py index e3c81a1f..4f09be47 100644 --- a/py-src/data_formulator/data_loader/mysql_data_loader.py +++ b/py-src/data_formulator/data_loader/mysql_data_loader.py @@ -129,6 +129,13 @@ def _read_sql(self, query: str) -> pa.Table: conn = self._get_conn() cur = conn.cursor() try: + # Force a fresh REPEATABLE READ snapshot on every read so rows + # committed by other sessions after this loader was first + # instantiated are visible without restarting the process. + try: + conn.commit() + except Exception: + pass cur.execute(query) if cur.description is None: return pa.table({}) diff --git a/py-src/data_formulator/routes/agents.py b/py-src/data_formulator/routes/agents.py index 950feeaa..2adfd4d3 100644 --- a/py-src/data_formulator/routes/agents.py +++ b/py-src/data_formulator/routes/agents.py @@ -360,6 +360,7 @@ def analyst_streaming(): other_threads = content.get("other_threads", None) primary_tables = content.get("primary_tables", None) attached_images = content.get("attached_images", None) + scratch_files = content.get("scratch_files", None) charts = content.get("charts", None) resume_trajectory = content.get("trajectory", None) completed_step_count = content.get("completed_step_count", 0) @@ -427,6 +428,7 @@ def generate(): primary_tables=primary_tables, attached_images=attached_images, charts=charts, + scratch_files=scratch_files, ): yield json.dumps(event, ensure_ascii=False) + '\n' diff --git a/py-src/data_formulator/routes/logs.py b/py-src/data_formulator/routes/logs.py new file mode 100644 index 00000000..5377c5e4 --- /dev/null +++ b/py-src/data_formulator/routes/logs.py @@ -0,0 +1,116 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +""" +Server log inspection routes. + +Data Formulator persists all server + Python-execution logs to a rotating +file under ``/logs/data_formulator.log`` (configured in +``app.configure_file_logging``). This is the artifact a user can send when +reporting a problem. + +Access policy — logs are **server-side only**: + +* In **local single-user mode** (``is_local_mode()`` is true) the user *is* + the server operator, so these endpoints expose the log to the UI (view / + tail / download). +* In any **hosted / multi-user** deployment these endpoints return + ``ACCESS_DENIED``. The operator reads the file directly on the server host; + end users never see server logs. + +Routes: + GET /api/logs/info — metadata: path, size, existence, local-mode flag + GET /api/logs/tail — last N lines of the current log file + GET /api/logs/download — download the current log file +""" + +import logging +import os +from collections import deque + +from flask import Blueprint, current_app, request, send_file + +from data_formulator.auth.identity import is_local_mode +from data_formulator.error_handler import json_ok +from data_formulator.errors import AppError, ErrorCode + +logger = logging.getLogger(__name__) + +logs_bp = Blueprint("logs", __name__, url_prefix="/api/logs") + +# Hard cap so a huge log can never blow up memory / the response. +_MAX_TAIL_LINES = 5000 +_DEFAULT_TAIL_LINES = 500 + + +def _require_local_mode() -> None: + """Reject the request unless running in local single-user mode.""" + if not is_local_mode(): + raise AppError( + ErrorCode.ACCESS_DENIED, + "Server logs are only viewable in local mode.", + ) + + +def _log_path() -> str | None: + """Resolve the active log file path (set by configure_file_logging).""" + return current_app.config.get("LOG_FILE_PATH") + + +@logs_bp.route("/info", methods=["GET"]) +def logs_info(): + """Return log file metadata (local mode only).""" + _require_local_mode() + path = _log_path() + exists = bool(path) and os.path.isfile(path) + size = os.path.getsize(path) if exists else 0 + return json_ok({ + "path": path, + "exists": exists, + "size": size, + }) + + +@logs_bp.route("/tail", methods=["GET"]) +def logs_tail(): + """Return the last N lines of the current log file (local mode only).""" + _require_local_mode() + path = _log_path() + if not path or not os.path.isfile(path): + return json_ok({"path": path, "exists": False, "lines": [], "content": ""}) + + try: + lines = int(request.args.get("lines", _DEFAULT_TAIL_LINES)) + except (TypeError, ValueError): + lines = _DEFAULT_TAIL_LINES + lines = max(1, min(lines, _MAX_TAIL_LINES)) + + # deque with maxlen keeps memory bounded regardless of file size. + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + tail = deque(fh, maxlen=lines) + except OSError as exc: + raise AppError(ErrorCode.VALIDATION_ERROR, f"Could not read log file: {exc}") + + content = "".join(tail) + return json_ok({ + "path": path, + "exists": True, + "lines": len(tail), + "content": content, + }) + + +@logs_bp.route("/download", methods=["GET"]) +def logs_download(): + """Download the current log file as an attachment (local mode only).""" + _require_local_mode() + path = _log_path() + if not path or not os.path.isfile(path): + raise AppError(ErrorCode.VALIDATION_ERROR, "No log file available.") + return send_file( + path, + mimetype="text/plain", + as_attachment=True, + download_name="data_formulator.log", + ) diff --git a/py-src/data_formulator/workflows/chart_semantics.py b/py-src/data_formulator/workflows/chart_semantics.py index 1b87f51b..7d3734ac 100644 --- a/py-src/data_formulator/workflows/chart_semantics.py +++ b/py-src/data_formulator/workflows/chart_semantics.py @@ -11,7 +11,7 @@ - VL type resolution (nominal / ordinal / temporal / quantitative) - Ordinal sort order (months, days, quarters) -This is intentionally minimal. The TS agents-chart library is the +This is intentionally minimal. The flint-chart TS library is the canonical source of truth for formatting, color schemes, tick constraints, domain constraints, and other visual refinements. The Python side focuses on getting the structural type decisions right diff --git a/pyproject.toml b/pyproject.toml index e4a6ea56..5bae92b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "data_formulator" -version = "0.8.0a1" +version = "0.8.0a2" requires-python = ">=3.11" authors = [ @@ -27,7 +27,11 @@ dependencies = [ "flask-limiter", "openai", "python-dotenv", - "litellm", + # litellm 1.92+ switched to a Rust/maturin build and ships manylinux-only + # wheels (no win_amd64 / macosx / py3-none-any), so installs hang on + # Windows/macOS without a Rust toolchain. Pin to the last universal-wheel + # line; >=1.84.0 keeps the litellm CVE fix and allows aiohttp>=3.14. + "litellm>=1.84.0,<1.92", "duckdb", "numpy", "vl-convert-python", diff --git a/requirements.txt b/requirements.txt index f7ef217e..c273db1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,9 @@ flask-cors flask-limiter openai python-dotenv -litellm +# litellm 1.92+ is Rust/maturin (manylinux-only wheels) — hangs on Windows/macOS +# without a Rust toolchain. Pin below it; >=1.84.0 keeps the CVE fix. +litellm>=1.84.0,<1.92 duckdb numpy vl-convert-python diff --git a/src/app/App.tsx b/src/app/App.tsx index c97f4bd6..700ba6ef 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -67,7 +67,7 @@ import { useWorkspaceAutoName } from './useWorkspaceAutoName'; import GridViewIcon from '@mui/icons-material/GridView'; import ViewSidebarIcon from '@mui/icons-material/ViewSidebar'; -import SettingsIcon from '@mui/icons-material/Settings'; +import SettingsOutlinedIcon from '@mui/icons-material/SettingsOutlined'; import { createBrowserRouter, Link as RouterLink, @@ -84,6 +84,7 @@ import { AppDispatch } from './store'; import dfLogo from '../assets/df-logo.png'; import { AnvilLoader } from '../components/AnvilLoader'; import { ModelSelectionButton } from '../views/ModelSelectionDialog'; +import { LogViewerDialog } from '../views/LogViewerDialog'; import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'; import UploadFileIcon from '@mui/icons-material/UploadFile'; import DownloadIcon from '@mui/icons-material/Download'; @@ -106,7 +107,7 @@ import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; import YouTubeIcon from '@mui/icons-material/YouTube'; import PublicIcon from '@mui/icons-material/Public'; import { useTranslation } from 'react-i18next'; -import { syncVegaLocale } from '../lib/vega-locale'; +import { syncVegaLocale } from '../i18n/vega-locale'; // Discord Icon Component const DiscordIcon: FC<{ sx?: any }> = ({ sx }) => ( @@ -206,12 +207,16 @@ const LanguageSwitcher: React.FC = () => { sx={{ height: '28px', my: 'auto', - mr: 1, '& .MuiToggleButton-root': { textTransform: 'none', fontSize: '12px', py: 0, minWidth: '40px', + color: 'text.secondary', + borderColor: 'divider', + '&.Mui-selected': { + color: 'text.primary', + }, }, }} > @@ -465,8 +470,18 @@ const ExitSessionButton: React.FC = () => { size="small" variant="text" onClick={handleExit} - startIcon={} - sx={{ textTransform: 'none', color: 'text.secondary' }} + startIcon={} + sx={{ + textTransform: 'none', + fontSize: '13px', + fontWeight: 400, + px: 1.5, + py: 0.5, + minWidth: 'auto', + lineHeight: 1.5, + color: 'text.secondary', + '&:hover': { color: 'text.primary', backgroundColor: 'rgba(0, 0, 0, 0.04)' }, + }} > {t('workspace.exit', { defaultValue: 'Exit' })} @@ -487,7 +502,7 @@ const ConfigDialog: React.FC = () => { const [formulateTimeoutSeconds, setFormulateTimeoutSeconds] = useState(config.formulateTimeoutSeconds ?? 180); const [defaultChartWidth, setDefaultChartWidth] = useState(config.defaultChartWidth ?? 300); const [defaultChartHeight, setDefaultChartHeight] = useState(config.defaultChartHeight ?? 300); - const [maxStretchFactor, setMaxStretchFactor] = useState(config.maxStretchFactor ?? 2.0); + const [maxStretchFactor, setMaxStretchFactor] = useState(config.maxStretchFactor ?? 1.5); const [frontendRowLimit, setFrontendRowLimit] = useState(config.frontendRowLimit ?? rowLimitDefault); const [paletteKey, setPaletteKey] = useState( (config.paletteKey && palettes[config.paletteKey]) ? config.paletteKey : defaultPaletteKey @@ -502,9 +517,20 @@ const ConfigDialog: React.FC = () => { return ( <> - + + setOpen(true)} + aria-label={t('app.settings')} + sx={{ + p: 0.5, + color: 'text.secondary', + '&:hover': { color: 'text.primary', backgroundColor: 'rgba(0, 0, 0, 0.04)' }, + }} + > + + + setOpen(false)} open={open}> {t('app.settings')} @@ -745,9 +771,9 @@ const AppShell: FC = () => { const viewMode = useSelector((state: DataFormulatorState) => state.viewMode); const tables = useSelector((state: DataFormulatorState) => state.tables); const activeWorkspace = useSelector((state: DataFormulatorState) => state.activeWorkspace); + const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); - useEffect(() => { - const authError = searchParams.get('auth_error'); + useEffect(() => { const authError = searchParams.get('auth_error'); if (!authError) return; const i18nKey = AUTH_ERROR_MESSAGES[authError] || 'auth.ssoErrorGeneric'; dispatch(dfActions.addMessages({ @@ -841,14 +867,34 @@ const AppShell: FC = () => { )} {isAppPage && ( - - - - + + + + + + {serverConfig.IS_LOCAL_MODE && } + + + + + + {activeWorkspace && ( <> - + )} @@ -927,25 +973,6 @@ const AppShell: FC = () => { )} - {isAppPage && ( - - - - )} diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 581ea618..33914c3c 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -2,12 +2,13 @@ // Licensed under the MIT License. import { createAsyncThunk, createSlice, PayloadAction, createSelector } from '@reduxjs/toolkit' -import { Channel, Chart, ChartTemplate, DataCleanBlock, DataSourceConfig, EncodingItem, EncodingMap, FieldItem, Trigger, ChartStyleVariant, DraftNode, InteractionEntry, DeriveStatus, ChatMessage, PendingTableLoad, PendingClarification } from '../components/ComponentType' +import { Channel, Chart, ChartTemplate, DataCleanBlock, DataSourceConfig, EncodingItem, EncodingMap, FieldItem, Trigger, ChartStyleVariant, DraftNode, InteractionEntry, DeriveStatus, ChatMessage, PendingTableLoad, PendingClarification, TextTurn } from '../components/ComponentType' import { enableMapSet } from 'immer'; import { DictTable } from "../components/ComponentType"; import { Message } from '../views/MessageSnackbar'; import { getChartTemplate, getChartChannels } from "../components/ChartTemplates" import { vlAdaptChart, vlRecommendEncodings } from 'flint-chart'; +import { migrateState } from './stateMigrations'; import { getDataTable } from '../views/ChartUtils'; import { getTriggers, getUrls, computeContentHash } from './utils'; import { apiRequest } from './apiClient'; @@ -104,6 +105,7 @@ export type FocusedId = | { type: 'table'; tableId: string } | { type: 'chart'; chartId: string } | { type: 'report'; reportId: string } + | { type: 'text'; textId: string } | undefined; export const DEFAULT_ROW_LIMIT = 2_000_000; @@ -113,7 +115,7 @@ export interface ClientConfig { formulateTimeoutSeconds: number; defaultChartWidth: number; defaultChartHeight: number; - maxStretchFactor: number; // max per-axis stretch multiplier for chart sizing (default 2.0) + maxStretchFactor: number; // max per-axis stretch multiplier for chart sizing (default 1.5) frontendRowLimit: number; // max rows to keep in browser when loading locally (non-virtual) paletteKey: string; // active color palette key from tokens.ts miniMode: boolean; // when true, run the single-turn MiniAnalystAgent (for small/local models) @@ -257,6 +259,9 @@ export interface DataFormulatorState { // Generated reports state generatedReports: GeneratedReport[]; + // Text turns (clarify / explain) placed in a thread by their one authored + // edge `parentNodeId` — the node the user asked from (design-docs/42). + textTurns: TextTurn[]; // Session loading overlay sessionLoading: boolean; @@ -333,7 +338,7 @@ const initialState: DataFormulatorState = { formulateTimeoutSeconds: 180, defaultChartWidth: 400, defaultChartHeight: 300, - maxStretchFactor: 2.0, + maxStretchFactor: 1.5, frontendRowLimit: DEFAULT_ROW_LIMIT, paletteKey: 'fluent', miniMode: false, @@ -352,6 +357,7 @@ const initialState: DataFormulatorState = { agentHandoffRequest: null, generatedReports: [], + textTurns: [], sessionLoading: false, sessionLoadingLabel: '', @@ -549,6 +555,10 @@ let removeTableStateRoutine = (state: DataFormulatorState, tableId: string) => { // Delete reports triggered from this table state.generatedReports = state.generatedReports.filter(r => r.triggerTableId !== tableId); + // Delete text turns anchored to this table (design-docs/42): those whose + // authored thread edge points directly at it (`parentNodeId === tableId`). + state.textTurns = state.textTurns.filter(a => a.parentNodeId !== tableId); + // Drop this table's starter questions / generation status delete state.starterQuestions[tableId]; delete state.starterQuestionsStatus[tableId]; @@ -764,6 +774,7 @@ export const dataFormulatorSlice = createSlice({ state.dataLoadingChatPending = null; state.generatedReports = []; + state.textTurns = []; // Clear active workspace so stale IDs don't persist across restarts state.activeWorkspace = null; @@ -836,7 +847,11 @@ export const dataFormulatorSlice = createSlice({ state.starterQuestionsStatus[action.payload.tableId] = 'error'; }, loadState: (state, action: PayloadAction) => { - const saved = action.payload; + // Upgrade an older persisted payload through the versioned migration + // chain before applying it (see stateMigrations.ts). Idempotent + // field backfills below handle "new optional field" cases that need + // no version. + const saved = migrateState(action.payload); // Return a brand-new state object so Immer skips // recursive proxy / freeze on potentially huge table rows. @@ -906,6 +921,7 @@ export const dataFormulatorSlice = createSlice({ dataLoadingChatMessages: saved.dataLoadingChatMessages || [], dataLoadingChatPending: null, generatedReports: saved.generatedReports || [], + textTurns: saved.textTurns || [], // Reset transient fields messages: [], @@ -1627,6 +1643,18 @@ export const dataFormulatorSlice = createSlice({ ]; } }, + // Remove specific interaction entries (by timestamp) from a table's + // trigger interaction — used to delete a resolved conversation block + // (e.g. an explanation) from the data thread. + removeInteractionEntries: (state, action: PayloadAction<{ tableId: string; timestamps: number[] }>) => { + const table = state.tables.find(t => t.id === action.payload.tableId); + if (table?.derive?.trigger?.interaction) { + const ts = new Set(action.payload.timestamps); + table.derive.trigger.interaction = table.derive.trigger.interaction.filter( + e => e.timestamp === undefined || !ts.has(e.timestamp) + ); + } + }, overrideDerivedTables: (state, action: PayloadAction) => { let table = action.payload; @@ -1915,6 +1943,52 @@ export const dataFormulatorSlice = createSlice({ clearAgentHandoffRequest: (state) => { state.agentHandoffRequest = null; }, + // ── Text turns (clarify / explain) — design-docs/41 ── + addTextTurn: (state, action: PayloadAction) => { + const turn = action.payload; + const existingIndex = state.textTurns.findIndex(a => a.id === turn.id); + if (existingIndex >= 0) { + state.textTurns[existingIndex] = turn; + } else { + state.textTurns.push(turn); + } + }, + updateTextTurn: (state, action: PayloadAction<{ id: string } & Partial>) => { + const { id, ...patch } = action.payload; + const turn = state.textTurns.find(a => a.id === id); + if (turn) Object.assign(turn, patch); + }, + removeTextTurn: (state, action: PayloadAction) => { + const turnId = action.payload; + const turn = state.textTurns.find(a => a.id === turnId); + const wasFocused = state.focusedId?.type === 'text' && state.focusedId.textId === turnId; + state.textTurns = state.textTurns.filter(a => a.id !== turnId); + // Fallback focus (design-docs/42): the source chart it came from, + // else its thread-parent table (walk parentNodeId), else nothing. + if (wasFocused && turn) { + const allCharts = collectAllCharts(state); + if (turn.sourceChartId && allCharts.some(c => c.id === turn.sourceChartId)) { + state.focusedId = { type: 'chart', chartId: turn.sourceChartId }; + } else { + // Resolve the turn's thread position to a table id. + let parentTableId: string | undefined; + let cur: TextTurn | undefined = turn; + const seen = new Set(); + while (cur && !seen.has(cur.id)) { + seen.add(cur.id); + const p: string | undefined = cur.parentNodeId; + if (!p) break; + if (state.tables.some(t => t.id === p)) { parentTableId = p; break; } + cur = state.textTurns.find(tt => tt.id === p); + } + if (parentTableId) { + state.focusedId = { type: 'table', tableId: parentTableId }; + } else { + state.focusedId = undefined; + } + } + } + }, // Generated reports actions saveGeneratedReport: (state, action: PayloadAction) => { const report = action.payload; @@ -2316,11 +2390,71 @@ export const dfSelectors = { getEffectiveTableId: (state: DataFormulatorState): string | undefined => { if (!state.focusedId) return undefined; if (state.focusedId.type === 'table') return state.focusedId.tableId; + // A focused text artifact is non-canvas-owning (design-docs/41): resolve + // it to its source chart's table, else its thread-parent table. + if (state.focusedId.type === 'text') { + const textId = state.focusedId.textId; + const art = state.textTurns.find(a => a.id === textId); + if (!art) return undefined; + if (art.sourceChartId) { + const c = collectAllCharts(state).find(ch => ch.id === art.sourceChartId); + if (c) return c.tableRef; + } + // Walk parentNodeId to the turn's thread-parent table (design-docs/42). + let cur: TextTurn | undefined = art; + const seen = new Set(); + while (cur && !seen.has(cur.id)) { + seen.add(cur.id); + const p: string | undefined = cur.parentNodeId; + if (!p) break; + if (state.tables.some(t => t.id === p)) return p; + cur = state.textTurns.find(tt => tt.id === p); + } + return undefined; + } // type === 'chart': derive table from the chart's tableRef let allCharts = collectAllCharts(state); let chart = allCharts.find(c => c.id === (state.focusedId as { type: 'chart'; chartId: string }).chartId); return chart?.tableRef; }, + /** + * The focus the CANVAS should render for. A focused text artifact is + * non-canvas-owning (design-docs/41): it overlays a panel above the chat + * while the canvas keeps showing the artifact's source chart (or, when no + * chart exists, its source table). Every other focus type passes through + * unchanged, so canvas consumers can read this in place of raw `focusedId`. + * Memoized so consuming components don't re-render on unrelated dispatches. + */ + selectCanvasTarget: createSelector( + [ + (state: DataFormulatorState) => state.focusedId, + (state: DataFormulatorState) => state.textTurns, + (state: DataFormulatorState) => state.charts, + selectTriggerCharts, + (state: DataFormulatorState) => state.tables, + ], + (focusedId, textTurns, userCharts, triggerCharts, tables): FocusedId => { + if (focusedId?.type !== 'text') return focusedId; + const art = textTurns.find(a => a.id === focusedId.textId); + if (!art) return undefined; + if (art.sourceChartId + && [...userCharts, ...triggerCharts].some(c => c.id === art.sourceChartId)) { + return { type: 'chart', chartId: art.sourceChartId }; + } + // No source chart: resolve the turn's thread position to a table + // (walk parentNodeId) and keep it on the canvas (design-docs/42). + let cur: TextTurn | undefined = art; + const seen = new Set(); + while (cur && !seen.has(cur.id)) { + seen.add(cur.id); + const p: string | undefined = cur.parentNodeId; + if (!p) break; + if (tables.some(t => t.id === p)) return { type: 'table', tableId: p }; + cur = textTurns.find(tt => tt.id === p); + } + return undefined; + }, + ), getFocusedChartId: (state: DataFormulatorState): string | undefined => { return state.focusedId?.type === 'chart' ? state.focusedId.chartId : undefined; }, diff --git a/src/app/stateMigrations.ts b/src/app/stateMigrations.ts new file mode 100644 index 00000000..6e8b0a45 --- /dev/null +++ b/src/app/stateMigrations.ts @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Versioned migrations for the persisted Data Formulator state + * (`session_state.json` on the server, IndexedDB for ephemeral, and exported + * workspace zips — all the same JSON payload). + * + * `getSerializableState` stamps the current `DF_STATE_VERSION` as + * `__stateVersion` on every save. `migrateState`, run at the top of the + * `loadState` reducer, upgrades an older saved payload through an ordered chain + * of pure transforms before it reaches the store. Sessions written before this + * scheme carry no `__stateVersion` and are treated as version 0. + * + * Conventions: + * - Forward-only. Each migration bumps the version by 1 and is a pure + * `(state) => state` (no side effects, returns a new-enough object). + * - Idempotent field backfills that only ADD optional fields (e.g. `virtual`, + * `config` defaults, stripping a legacy field) stay inline in `loadState` — + * they tolerate any input and need no version. Use a numbered migration here + * only for STRUCTURAL rewrites that field presence can't express (e.g. + * moving data between collections, renaming a shape). + * - Bump `DF_STATE_VERSION` to match the highest `to` you add below. + * - No downgrade path: a state newer than this client is returned untouched + * (a "session is newer" guard can be added later if needed). + */ + +/** Current persisted-state schema version. Bump when adding a migration. */ +export const DF_STATE_VERSION = 1; + +type SavedState = Record; + +interface Migration { + /** The version this migration produces; applied when `saved < to`. */ + to: number; + migrate: (state: SavedState) => SavedState; +} + +/** + * Ordered chain of structural migrations. Append new entries with the next + * integer `to` and bump `DF_STATE_VERSION` to match. Example (design-docs/41): + * { to: 1, migrate: s => convertInteractionExchangesToMessages(s) } + */ +const MIGRATIONS: Migration[] = [ + { + // design-docs/42: text turns gain an authored thread edge + // (`parentNodeId`). Backfill it from the legacy anchors so pre-42 + // sessions place turns correctly: a resolved turn follows the table it + // produced, else the table it derives from. + to: 1, + migrate: (s) => { + const turns = Array.isArray(s.textTurns) ? s.textTurns : undefined; + if (!turns) return s; + return { + ...s, + textTurns: turns.map((tt: any) => + tt && tt.parentNodeId == null + ? { ...tt, parentNodeId: tt.resultTableId ?? tt.sourceTableId } + : tt, + ), + }; + }, + }, +]; + +/** + * Upgrade a saved state payload to `DF_STATE_VERSION`. Unversioned payloads are + * treated as version 0. Returns the (possibly transformed) payload; the + * `loadState` reducer then applies its idempotent field backfills on top. + */ +export function migrateState(saved: SavedState | null | undefined): SavedState { + if (!saved || typeof saved !== 'object') return saved ?? {}; + let from = typeof saved.__stateVersion === 'number' ? saved.__stateVersion : 0; + if (from >= DF_STATE_VERSION) return saved; + let migrated = saved; + for (const m of MIGRATIONS) { + if (m.to > from && m.to <= DF_STATE_VERSION) { + migrated = m.migrate(migrated); + from = m.to; + } + } + return migrated; +} diff --git a/src/app/useAutoSave.tsx b/src/app/useAutoSave.tsx index e2888a17..0fe9d42e 100644 --- a/src/app/useAutoSave.tsx +++ b/src/app/useAutoSave.tsx @@ -6,6 +6,7 @@ import { useSelector } from 'react-redux'; import { DataFormulatorState } from './dfSlice'; import { saveWorkspaceState } from './workspaceService'; import { handleApiError } from './errorHandler'; +import { DF_STATE_VERSION } from './stateMigrations'; /** * Fields excluded from auto-save (secrets / ephemeral / fetched-on-startup). @@ -39,6 +40,9 @@ export function getSerializableState(state: DataFormulatorState): Record + + + + + + + + + + + + + + + + + + diff --git a/src/assets/chart-icon-connected-scatter.svg b/src/assets/chart-icon-connected-scatter.svg new file mode 100644 index 00000000..e38e99ec --- /dev/null +++ b/src/assets/chart-icon-connected-scatter.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + diff --git a/src/assets/chart-icon-ecdf.svg b/src/assets/chart-icon-ecdf.svg new file mode 100644 index 00000000..6c8ac163 --- /dev/null +++ b/src/assets/chart-icon-ecdf.svg @@ -0,0 +1,10 @@ + + + + + + + + diff --git a/src/assets/chart-icon-gantt.svg b/src/assets/chart-icon-gantt.svg new file mode 100644 index 00000000..454c5949 --- /dev/null +++ b/src/assets/chart-icon-gantt.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/src/assets/chart-icon-placeholder.svg b/src/assets/chart-icon-placeholder.svg new file mode 100644 index 00000000..78715e9f --- /dev/null +++ b/src/assets/chart-icon-placeholder.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/src/assets/chart-icon-range-area.svg b/src/assets/chart-icon-range-area.svg new file mode 100644 index 00000000..a7d39e3d --- /dev/null +++ b/src/assets/chart-icon-range-area.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + diff --git a/src/assets/chart-icon-slope.svg b/src/assets/chart-icon-slope.svg new file mode 100644 index 00000000..88dc6ca1 --- /dev/null +++ b/src/assets/chart-icon-slope.svg @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/assets/chart-icon-sparkline.svg b/src/assets/chart-icon-sparkline.svg new file mode 100644 index 00000000..d4eda542 --- /dev/null +++ b/src/assets/chart-icon-sparkline.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/assets/chart-icon-violin.svg b/src/assets/chart-icon-violin.svg new file mode 100644 index 00000000..32eae7b6 --- /dev/null +++ b/src/assets/chart-icon-violin.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + diff --git a/src/components/ChartTemplates.tsx b/src/components/ChartTemplates.tsx index b66d61a6..265e2be1 100644 --- a/src/components/ChartTemplates.tsx +++ b/src/components/ChartTemplates.tsx @@ -50,6 +50,19 @@ import chartIconBump from '../assets/chart-icon-bump.svg'; import chartIconRose from '../assets/chart-icon-rose.svg'; import chartIconBarTable from '../assets/chart-icon-bar-table.svg'; import chartIconKpiCard from '../assets/chart-icon-kpi-card.svg'; +// Borrowed from the flint-chart gallery (same flat line-art design language) for +// types that previously fell back to the generic placeholder. +import chartIconConnectedScatter from '../assets/chart-icon-connected-scatter.svg'; +import chartIconGantt from '../assets/chart-icon-gantt.svg'; +import chartIconBullet from '../assets/chart-icon-bullet.svg'; +import chartIconEcdf from '../assets/chart-icon-ecdf.svg'; +import chartIconViolin from '../assets/chart-icon-violin.svg'; +import chartIconSlope from '../assets/chart-icon-slope.svg'; +import chartIconSparkline from '../assets/chart-icon-sparkline.svg'; +import chartIconRangeArea from '../assets/chart-icon-range-area.svg'; +// Generic fallback for chart types without a bespoke icon (matches the flat +// line-art language of the others: dark axes + muted bars + a blue trend line). +import chartIconPlaceholder from '../assets/chart-icon-placeholder.svg'; // Chart Icon Component using static imports const ChartIcon: React.FC<{ src: string; alt?: string }> = ({ src, alt = "" }) => { @@ -88,6 +101,17 @@ export const CHART_ICONS: Record = { "Strip Plot": , "Radar Chart": , "KPI Card": , + // Borrowed from the flint-chart gallery (see imports above). + "Connected Scatter Plot": , + "Gantt Chart": , + "Bullet Chart": , + "ECDF Plot": , + "Violin Plot": , + "Slope Chart": , + "Sparkline": , + "Range Area Chart": , + "Map": , + "Choropleth": , "Custom Point": , "Custom Line": , "Custom Bar": , @@ -102,7 +126,7 @@ export const CHART_ICONS: Record = { function addIcons(defs: { chart: string }[]): ChartTemplate[] { return defs.map(def => ({ ...def, - icon: CHART_ICONS[def.chart] || , + icon: CHART_ICONS[def.chart] || , })) as ChartTemplate[]; } diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index 84542d9c..103c9a5d 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -67,6 +67,10 @@ export interface InteractionEntry { plan?: string; // agent's reasoning / thought for this action content: string; displayContent?: string; + /** Names of files / images the user attached with this prompt, surfaced as + * chips in the message bubble (the file bytes live in workspace scratch/, + * not here). */ + attachments?: string[]; inputTableNames?: string[]; // table names actually used for this derivation step clarificationQuestions?: ClarificationQuestion[]; /** For 'delegate' entries: which peer agent the Data Agent wants to @@ -108,6 +112,55 @@ export interface DraftNode { export type ThreadNode = DraftNode | DictTable; +/** + * A first-class **text turn** (clarify / explain) in the thread — a sibling to + * charts, tables, and reports (see design-docs/41). Placed in the thread by its + * one authored edge, `parentNodeId` (design-docs/42): the node the user was + * asking from when the turn was created. Focusing one overlays a panel above the + * chat without taking over the canvas (the canvas keeps showing `sourceChartId`); + * deleting one is a generic artifact delete. (Delegate is NOT a text turn — a + * hand-off is an agent action, handled directly.) + */ +export interface TextTurn { + kind: 'text'; + id: string; + displayId: string; + /** clarify carries `options`; explain has none. */ + textKind: 'clarify' | 'explain'; + /** Markdown: the question preamble, or the answer. */ + content: string; + /** The user message that triggered this turn (shown with the card so the + * exchange stays self-contained — the run produced no table to anchor it). */ + prompt?: string; + /** clarify only (empty/undefined ⇒ a plain explanation). */ + options?: ClarificationQuestion[]; + /** True once the user has responded to THIS clarify — it then locks + * (read-only). A later response is a *new* conversation, not a re-answer. */ + answered?: boolean; + /** The user's response to this clarify, shown in the resolved read-only view + * (question → answer). */ + answer?: string; + /** + * The thread node this turn FOLLOWS (design-docs/42) — the SINGLE authored + * edge that places the turn in the thread. Captured once at ask time: the + * table the user asked from (a fresh turn), or the previous node in the run + * (a chained follow-up / the turn being answered). Branching = two nodes + * sharing a parent. This fully replaces the old inferred positioning + * (sourceTableId / resultTableId / conversationId), which are now deprecated. + */ + parentNodeId: string; + /** The chart the user was on when this turn was created — canvas provenance + * only (focusing the turn keeps this chart on the canvas). NOT positioning. */ + sourceChartId?: string; + actionId?: string; + /** + * §12 opaque resume token — set iff the backend stamped a trajectory on the + * emitting event (continuation opt-in). Absent ⇒ followups are fresh turns. + */ + resume?: { trajectory: any[]; completedStepCount: number }; + createdAt: number; +} + // Define data cleaning message types export type DataCleanTableOutput = { name: string; @@ -301,6 +354,16 @@ export interface DictTable { anchored: boolean; // whether this table is anchored as a persistent table used to derive other tables description: string; // table-level description sourced from the loader (read-only). Empty string when none. + /** + * Authored THREAD edge (design-docs/42): the node this table FOLLOWS in the + * thread — set to a text-turn id when a conversation produced the table (the + * chain `table → clarifyTurn → askedFromTable`). It overrides + * `derive.trigger.tableId` for POSITION only; data provenance + * (`derive.source` / `derive.trigger`) is unchanged. Unset ⇒ the table's + * thread parent is its data parent (`derive.trigger.tableId`). + */ + threadParentId?: string; + source?: DataSourceConfig; // Content hash for detecting data changes during refresh diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 44c8109c..86c12c61 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -47,6 +47,13 @@ "data": "Data", "microsoftResearch": "Microsoft Research" }, + "logs": { + "title": "Server Logs", + "viewLogs": "View server logs", + "refresh": "Refresh", + "download": "Download full log", + "empty": "Log file is empty." + }, "session": { "exportSession": "export session", "importSession": "import session", @@ -423,6 +430,9 @@ "threadIndex": "thread - {{index}}", "continuedFromAbove": "continued", "continuesBelow": "continues", + "textTurnEarlier": "{{count}} earlier reply", + "textTurnEarlier_other": "{{count}} earlier replies", + "textTurnCollapse": "Collapse", "usingSources": "Using", "hmm": "hmm...", "oops": "oops...", @@ -526,6 +536,7 @@ "placeholderFormulateEmphasis": "✏️ formulate data", "formulateAndOverride": "formulate and override", "agentWorking": "Agent is working...", + "attachUploadFailed": "Failed to attach {{name}}", "replyPlaceholder": "Reply to agent's question...", "explorePlaceholder": "Ask questions or describe what to explore (add context with @)", "explorePlaceholderSingleTable": "Ask questions or describe what to explore", @@ -559,6 +570,8 @@ "clarificationTitle": "Question", "minimizeClarification": "Minimize", "expandClarification": "Expand", + "pauseClose": "Close (switch focus)", + "pauseDelete": "Delete", "clarificationQuestionLabel": "{{index}}.", "optionalClarification": "(optional)", "freeTextClarificationPlaceholder": "Type your answer...", diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index a718906d..d99518ed 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -47,6 +47,13 @@ "data": "数据", "microsoftResearch": "微软研究院" }, + "logs": { + "title": "服务器日志", + "viewLogs": "查看服务器日志", + "refresh": "刷新", + "download": "下载完整日志", + "empty": "日志文件为空。" + }, "session": { "exportSession": "导出会话", "importSession": "导入会话", @@ -423,6 +430,9 @@ "threadIndex": "线程 - {{index}}", "continuedFromAbove": "续上", "continuesBelow": "续下", + "textTurnEarlier": "之前的 {{count}} 条回复", + "textTurnEarlier_other": "之前的 {{count}} 条回复", + "textTurnCollapse": "收起", "usingSources": "使用", "hmm": "嗯...", "oops": "出错了...", @@ -578,6 +588,7 @@ "placeholderFormulateEmphasis": "✏️ 整理数据", "formulateAndOverride": "整理并覆盖", "agentWorking": "Agent 努力工作中...", + "attachUploadFailed": "附加 {{name}} 失败", "replyPlaceholder": "回复 Agent 的问题...", "explorePlaceholder": "有什么问题，有什么想要探索的？（用 @ 添加上下文）", "explorePlaceholderSingleTable": "有什么问题，有什么想要探索的？", @@ -611,6 +622,8 @@ "clarificationTitle": "有问题", "minimizeClarification": "收起", "expandClarification": "展开", + "pauseClose": "关闭（切换焦点）", + "pauseDelete": "删除", "clarificationQuestionLabel": "{{index}}.", "optionalClarification": "（可选）", "freeTextClarificationPlaceholder": "输入你的回答...", diff --git a/src/lib/vega-locale.ts b/src/i18n/vega-locale.ts similarity index 78% rename from src/lib/vega-locale.ts rename to src/i18n/vega-locale.ts index f0004d64..ab16ca7e 100644 --- a/src/lib/vega-locale.ts +++ b/src/i18n/vega-locale.ts @@ -11,8 +11,8 @@ * Call {@link syncVegaLocale} once at startup and on every language change. */ -import { defaultLocale } from 'vega'; -import i18n from '../i18n'; +import { formatLocale, timeFormatLocale } from 'vega'; +import i18n from './index'; const D3_DEFAULT_NUMBER = { decimal: '.', @@ -42,5 +42,9 @@ function readTimeLocale(): Record | null { export function syncVegaLocale(): void { const timeLocale = readTimeLocale(); - defaultLocale(D3_DEFAULT_NUMBER as any, (timeLocale ?? D3_DEFAULT_TIME) as any); -} + // `vega` re-exports numberFormatDefaultLocale as formatLocale and + // timeFormatDefaultLocale as timeFormatLocale; both set the process-wide + // default locale (equivalent to the old combined `defaultLocale`). + formatLocale(D3_DEFAULT_NUMBER); + timeFormatLocale(timeLocale ?? D3_DEFAULT_TIME); +} \ No newline at end of file diff --git a/src/views/AgentChatInput.tsx b/src/views/AgentChatInput.tsx index 0c82c635..1ef24d6f 100644 --- a/src/views/AgentChatInput.tsx +++ b/src/views/AgentChatInput.tsx @@ -264,11 +264,13 @@ export const AgentChatInput: React.FC = ({ diff --git a/src/views/AgentPausePanel.tsx b/src/views/AgentPausePanel.tsx index 3f8b85e6..6ed267d4 100644 --- a/src/views/AgentPausePanel.tsx +++ b/src/views/AgentPausePanel.tsx @@ -22,14 +22,12 @@ import React, { FC, ReactNode, useEffect, useRef, useState } from 'react'; import { - Box, Collapse, IconButton, InputAdornment, TextField, Tooltip, Typography, useTheme, + Box, IconButton, InputAdornment, TextField, Tooltip, Typography, useTheme, } from '@mui/material'; import { alpha } from '@mui/material/styles'; import SearchIcon from '@mui/icons-material/Search'; import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; -import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; -import UnfoldLessIcon from '@mui/icons-material/UnfoldLess'; -import UnfoldMoreIcon from '@mui/icons-material/UnfoldMore'; +import CloseRoundedIcon from '@mui/icons-material/CloseRounded'; import ArrowForwardRoundedIcon from '@mui/icons-material/ArrowForwardRounded'; import CheckRoundedIcon from '@mui/icons-material/CheckRounded'; import { useTranslation } from 'react-i18next'; @@ -48,67 +46,53 @@ import { renderFieldHighlights, CompactMarkdown } from './InteractionEntryCard'; // --------------------------------------------------------------------------- interface AgentPauseShellProps { - /** Localized header label shown both when expanded and minimized. */ + /** Localized header label. */ title: string; - /** Short preview text shown next to the title when the panel is minimized. */ - minimizedPreview?: string; - /** Tooltip + behavior for the dismiss (×) icon. */ - dismissTooltip: string; - /** Tooltip labels for the minimize/expand toggle. */ - minimizeTooltip: string; - expandTooltip: string; + /** Tooltip for the close (×) icon — de-highlights / switches focus. */ + closeTooltip: string; + /** Deprecated: delete is handled from the thread card directly, so the + * panel no longer renders a delete button. Kept optional for callers. */ + deleteTooltip?: string; /** * Icon glyph rendered in the header. Callers pass a fully-styled * `AgentToyIcon` (or any node) so the shell stays agnostic of icon - * variants and colors. The shell only fades the icon when minimized. + * variants and colors. */ icon: ReactNode; /** - * Optional accent color driving the panel's chrome (bg fill, border, - * hover). When omitted the panel uses neutral greyscale chrome. Each - * pause variant passes its own semantic hue so clarify / explain / + * Optional accent color driving the panel's chrome (bg fill, border). + * When omitted the panel uses a faint wash of the theme's SECONDARY color. + * Each pause variant passes its own semantic hue so clarify / explain / * suggest panels read as visually distinct moments in the timeline * (parallel to the tinted bubbles in `InteractionEntryCard`). */ accentColor?: string; - /** Called when the user dismisses the pause. */ - onCancel: () => void; - /** Reset minimized state whenever this value changes (e.g. new questions). */ - resetKey?: unknown; + /** Close: de-highlight the pause and switch focus to the previous chart. */ + onClose: () => void; + /** Deprecated: delete is handled from the thread card directly. */ + onDelete?: () => void; children: ReactNode; } const AgentPauseShell: FC = ({ title, - minimizedPreview, - dismissTooltip, - minimizeTooltip, - expandTooltip, + closeTooltip, icon, accentColor, - onCancel, - resetKey, + onClose, children, }) => { const theme = useTheme(); - const [minimized, setMinimized] = useState(false); // Chrome is a soft tinted fill in the accent hue. When no explicit accent // is given the panel falls back to a faint wash of the theme's SECONDARY // color — a different, theme-derived hue from the primary blue used by the - // chat affordances, so the panel reads as its own subtle surface without - // echoing the primary CTA color. Interactive affordances (chips, CTAs) - // keep the primary hue for the strongest color. + // chat affordances, so the panel reads as its own subtle surface. const fillAccent = accentColor ?? theme.palette.secondary.main; const panelBg = alpha(fillAccent, 0.05); const panelBorder = alpha(fillAccent, 0.18); - const panelHover = alpha(fillAccent, 0.09); const primaryColor = theme.palette.primary.main; - // Reset minimize when the underlying pause changes so a brand-new - // pause shows up expanded by default. - useEffect(() => { setMinimized(false); }, [resetKey]); - return ( = ({ borderBottom: `1px solid ${panelBorder}`, backgroundColor: panelBg, // Inset slightly under the parent Card's gradient border (1.5px - // stroke + 12px outer radius → ~10.5px inner radius). Match - // that inner curve so the panel's rounded corners hug the - // gradient border instead of sticking out as squared edges. + // stroke + 12px outer radius → ~10.5px inner radius) so the panel's + // rounded corners hug the gradient border cleanly. borderRadius: '10.5px 10.5px 0 0', overflow: 'hidden', mx: '-10px', mt: '-8px', mb: '4px', }}> - setMinimized(prev => !prev)} - sx={{ - display: 'flex', alignItems: 'center', gap: '6px', minHeight: 16, - cursor: 'pointer', - // Stretch hover background to the panel's full content - // width by extending past the parent's px: 0.5 padding, - // then re-add it on the inside. Header owns the top - // spacing so the hover bg fills cleanly to the panel's - // rounded top edge. - px: 0.5, mx: -0.5, pt: '8px', pb: '6px', - '&:hover': { backgroundColor: panelHover }, - }} - > - + + {icon} - {minimized ? ( - - - {title} - - {minimizedPreview && ( - - {minimizedPreview.slice(0, 120)} - - )} - - ) : ( - - {title} - - )} - - { e.stopPropagation(); onCancel(); }} - sx={{ - p: 0, width: 16, height: 16, - color: theme.palette.text.disabled, - '&:hover': { color: theme.palette.error.main }, - }} - > - - - - + + {title} + + - {minimized - ? - : } + - - {children} - + {children} ); }; @@ -249,7 +172,10 @@ interface ClarificationPanelProps { /** Clear a question's recorded answer (e.g. the user edits its field). */ onClearAnswer?: (questionIndex: number) => void; onSubmit: (responses: ClarificationResponse[]) => void; - onCancel: () => void; + /** Close: de-highlight the pause and switch focus to the previous chart. */ + onClose: () => void; + /** Delete: remove this pending pause block. */ + onDelete: () => void; } export const ClarificationPanel: FC = ({ @@ -259,7 +185,8 @@ export const ClarificationPanel: FC = ({ onSelectAnswer, onClearAnswer, onSubmit, - onCancel, + onClose, + onDelete, }) => { const theme = useTheme(); const { t } = useTranslation(); @@ -495,12 +422,10 @@ export const ClarificationPanel: FC = ({ />} accentColor={chromeAccent} title={title} - minimizedPreview={questions[0]?.text || ''} - dismissTooltip={t('chartRec.cancelClarification')} - minimizeTooltip={t('chartRec.minimizeClarification')} - expandTooltip={t('chartRec.expandClarification')} - onCancel={onCancel} - resetKey={questions} + closeTooltip={t('chartRec.pauseClose')} + deleteTooltip={t('chartRec.pauseDelete')} + onClose={onClose} + onDelete={onDelete} > 1 ? '14px' : '4px', pb: '8px' }}> {questions.map((question, questionIndex) => { @@ -652,8 +577,10 @@ interface DelegatePanelProps { /** One or two hand-off option prompts (cards). Each string is shown * on the button and used as the seed prompt for the target agent. */ options: string[]; - /** Dismiss the suggestion (treated as cancelling the pause). */ - onCancel: () => void; + /** Close: de-highlight the suggestion and switch focus to the previous chart. */ + onClose: () => void; + /** Delete: remove this suggestion block. */ + onDelete: () => void; } /** @@ -668,7 +595,8 @@ export const DelegatePanel: FC = ({ target, message, options, - onCancel, + onClose, + onDelete, }) => { const theme = useTheme(); const { t } = useTranslation(); @@ -679,8 +607,8 @@ export const DelegatePanel: FC = ({ if (!cleanPrompt) return; dispatch(dfActions.requestAgentHandoff({ target, prompt: cleanPrompt })); // Hand off — the user's attention moves to the target agent - // and the data-agent run is done. - onCancel(); + // and the data-agent run is done, so the suggestion block is removed. + onDelete(); }; const isReport = target === 'report_gen'; @@ -702,12 +630,10 @@ export const DelegatePanel: FC = ({ />} accentColor={theme.palette.primary.main} title={t('chartRec.delegateTitle')} - minimizedPreview={message} - dismissTooltip={t('chartRec.delegateDismiss')} - minimizeTooltip={t('chartRec.delegateMinimize')} - expandTooltip={t('chartRec.delegateExpand')} - onCancel={onCancel} - resetKey={`${target}|${validOptions.join('||')}`} + closeTooltip={t('chartRec.pauseClose')} + deleteTooltip={t('chartRec.pauseDelete')} + onClose={onClose} + onDelete={onDelete} > = ({ interface ExplanationPanelProps { /** The agent's plain-text answer (markdown) to display read-only. */ content: string; - /** Dismiss the panel (the user is done reading the answer). */ - onCancel: () => void; + /** Close: de-highlight the panel and switch focus to the previous chart. */ + onClose: () => void; + /** Delete: remove this explanation block from the thread. */ + onDelete: () => void; } /** @@ -804,7 +732,7 @@ interface ExplanationPanelProps { * but carries no inputs or actions — it's purely "here's what I said", * dismissible by the header's delete button or by focusing another item. */ -export const ExplanationPanel: FC = ({ content, onCancel }) => { +export const ExplanationPanel: FC = ({ content, onClose, onDelete }) => { const theme = useTheme(); const { t } = useTranslation(); @@ -816,12 +744,10 @@ export const ExplanationPanel: FC = ({ content, onCancel />} accentColor={theme.palette.primary.main} title={t('chartRec.explanationTitle')} - minimizedPreview={content} - dismissTooltip={t('chartRec.dismissExplanation')} - minimizeTooltip={t('chartRec.minimizeClarification')} - expandTooltip={t('chartRec.expandClarification')} - onCancel={onCancel} - resetKey={content} + closeTooltip={t('chartRec.pauseClose')} + deleteTooltip={t('chartRec.pauseDelete')} + onClose={onClose} + onDelete={onDelete} > state.focusedId); let focusedChartId = focusedId?.type === 'chart' ? focusedId.chartId : undefined; + let textTurns = useSelector((state: DataFormulatorState) => state.textTurns); let focusedTableId = useMemo(() => { if (!focusedId) return undefined; if (focusedId.type === 'table') return focusedId.tableId; @@ -872,8 +875,28 @@ let SingleThreadGroupView: FC<{ const chart = charts.find(c => c.id === focusedId.chartId); return chart?.tableRef; } + if (focusedId.type === 'text') { + // Highlight the text turn's thread-parent table (or its source + // chart's table), mirroring chart focus (design-docs/41/42). + const turn = textTurns.find(tt => tt.id === focusedId.textId); + if (!turn) return undefined; + if (turn.sourceChartId) { + const chart = charts.find(c => c.id === turn.sourceChartId); + if (chart?.tableRef) return chart.tableRef; + } + let cur: TextTurn | undefined = turn; + const seen = new Set(); + while (cur && !seen.has(cur.id)) { + seen.add(cur.id); + const p: string | undefined = cur.parentNodeId; + if (!p) return undefined; + if (tableById.has(p)) return p; + cur = textTurns.find(tt => tt.id === p); + } + return undefined; + } return undefined; - }, [focusedId, charts]); + }, [focusedId, charts, textTurns]); let draftNodes = useSelector((state: DataFormulatorState) => state.draftNodes); let generatedReports = useSelector(dfSelectors.getAllGeneratedReports); @@ -890,7 +913,66 @@ let SingleThreadGroupView: FC<{ return map; }, [generatedReports]); - // Pre-index running/clarifying/completed status from DraftNodes + // Text turns render by their authored parent edge (design-docs/42): each + // turn is a child of its `parentNodeId` — a table (a fresh turn on it) or + // another turn (a chained follow-up). + const textTurnChildrenOf = useMemo(() => { + const map = new Map(); + for (const turn of textTurns) { + const key = turn.parentNodeId; + if (!key) continue; + const list = map.get(key) || []; + list.push(turn); + map.set(key, list); + } + for (const list of map.values()) list.sort((a, b) => (a.createdAt || 0) - (b.createdAt || 0)); + return map; + }, [textTurns]); + + const turnById = useMemo(() => new Map(textTurns.map(tt => [tt.id, tt])), [textTurns]); + + // A turn is a "lead-up" if it PRODUCED a table — i.e. it sits on some table's + // `threadParentId` chain (the clarify/answer that resolved into that table). + // Such turns render WITH their result table (as its lead-in, in the table's + // thread) so the conversation and its result stay connected — NOT at the root + // table. Terminal / still-pending turns (no result yet) render at the root's + // non-ghost card instead (design-docs/42). + const leadUpTurnIds = useMemo(() => { + const s = new Set(); + for (const t of tables) { + let cur: string | undefined = t.threadParentId; + const seen = new Set(); + while (cur && !seen.has(cur)) { + seen.add(cur); + const turn = turnById.get(cur); + if (!turn) break; // reached a table / unknown + s.add(turn.id); + cur = turn.parentNodeId; + if (cur && tableById.has(cur)) break; // reached the root table + } + } + return s; + }, [tables, turnById, tableById]); + + // The lead-up conversation for a table: the turn chain from its + // `threadParentId` up to (not including) the root table, oldest first. + const leadUpTurnsOf = (tableId: string): TextTurn[] => { + const t = tableById.get(tableId); + if (!t?.threadParentId) return []; + const out: TextTurn[] = []; + let cur: string | undefined = t.threadParentId; + const seen = new Set(); + while (cur && !seen.has(cur)) { + seen.add(cur); + const turn = turnById.get(cur); + if (!turn) break; + out.push(turn); + cur = turn.parentNodeId; + if (cur && tableById.has(cur)) break; + } + return out.reverse(); + }; + const runningAgentTableIds = useMemo(() => { const ids = new Map(); for (const d of draftNodes) { @@ -1294,7 +1376,7 @@ let SingleThreadGroupView: FC<{ key: `${keyPrefix}-conv-${tableId}-${ei}`, type: triggerType, highlighted, - element: , + element: , interactionEntry: pairs[pairs.length - 1].userEntry, gutterIcon: ( { + const isFocused = focusedId?.type === 'text' && focusedId.textId === turn.id; + const rowHL = highlighted || isFocused; + const preview = (turn.content || '') + .replace(/[#*`>|]/g, ' ').replace(/\s+/g, ' ').trim(); + // No timeline dot for text turns — instead an exchange (⇄) glyph sits + // OUTSIDE the box, to its left, flowing with the card (design-docs/41). + const gutterIcon = ; + const conversationIcon = ( + + ); + const card = ( + dispatch(dfActions.setFocused({ type: 'text', textId: turn.id }))} + > + + {showPrompt && turn.prompt && ( + + {turn.prompt} + + )} + + {preview} + + + {/* Delete floats over the top-right corner so it doesn't take + horizontal space from the text; a translucent bg + blur keeps + the trash icon readable over the content on hover. */} + + { e.stopPropagation(); dispatch(dfActions.removeTextTurn(turn.id)); }} + > + + + + + ); + // The follow-up reply renders as its OWN box below the card, using the + // user-response bubble style (InteractionEntryCard user prompt) so it + // reads as the user's turn (design-docs/41). + const answerBox = turn.answered && turn.answer ? ( + dispatch(dfActions.setFocused({ type: 'text', textId: turn.id }))} + > + + + ) : null; + const element = ( + + {conversationIcon} + + {card} + {answerBox} + + + ); + return { + key: `textturn-${turn.id}`, type: 'artifact' as const, highlighted: rowHL, + gutterIcon, element, + }; + }; + + // Render a single text turn: its triggering prompt bubble (if any) then the + // turn card. `keyNode` seeds prompt-entry keys. + const pushSingleTurn = (turn: TextTurn, keyNode: string, highlighted: boolean, triggerType: 'trigger' | 'leaf-trigger') => { + if (turn.prompt) { + pushInteractionEntries( + [{ from: 'user', to: 'data-agent', role: 'prompt', content: turn.prompt, timestamp: turn.createdAt }], + keyNode, triggerType, highlighted, `textturn-prompt-${turn.id}`, + ); + } + timelineItems.push(buildTextTurnTimelineItem(turn, highlighted, false)); + }; + + // Render the text-turn subtree rooted at a node (design-docs/42): the node's + // direct child turns in order, each followed by its own chained follow-ups + // (recursion). SKIPS lead-up turns — those produced a table and render WITH + // that result table (see leadUpTurnsOf / pushTableBlock), so here we render + // only the terminal / still-pending conversation on `nodeId`. + const pushTextTurnSubtree = (nodeId: string, highlighted: boolean, triggerType: 'trigger' | 'leaf-trigger') => { + const turns = textTurnChildrenOf.get(nodeId); + if (!turns) return; + for (const turn of turns) { + if (leadUpTurnIds.has(turn.id)) continue; // renders with its result table + pushSingleTurn(turn, nodeId, highlighted, triggerType); + // Chained follow-ups hang off this turn. + pushTextTurnSubtree(turn.id, highlighted, triggerType); + } + }; + + // Table-level entry point: render the turn subtree rooted at a table. Only + // ever called at the table's NON-GHOST card (design-docs/42), so a table shown + // as a used-parent ghost elsewhere renders no conversation there. + const pushTableTextTurns = (tableId: string, highlighted: boolean, triggerType: 'trigger' | 'leaf-trigger') => { + pushTextTurnSubtree(tableId, highlighted, triggerType); + }; + + // Render one table's full block in the thread body: its trigger interaction + // (split so the run's closing summary follows the outputs), the table card + + // charts, reports, the after-run summary, then any new conversation and the + // live agent draft. Shared by the new-table and leaf-table passes — they were + // identical apart from the trigger source and the card/type labels. (The old + // `afterTableMap`/`leafAfterTableMap` Maps were set and read in the same + // iteration, so they collapse to a local here.) + const pushTableBlock = ( + tableId: string, + trigger: Trigger | undefined, + tableCard: any, + triggerCardFallback: any, + tableType: 'table' | 'leaf-table', + triggerType: 'trigger' | 'leaf-trigger', + highlighted: boolean, + keyPrefix: string, + ) => { + // Lead-up conversation that PRODUCED this table (design-docs/42): the + // clarify/answer turns on its threadParentId chain, rendered BEFORE the + // trigger + card so the conversation and its result read as one thread. + for (const turn of leadUpTurnsOf(tableId)) { + pushSingleTurn(turn, tableId, highlighted, triggerType); + } + let afterEntries: InteractionEntry[] = []; + if (trigger) { + const interaction = trigger.interaction; + if (interaction && interaction.length > 0) { + const [before, after] = splitAtLastInstruction(interaction); + pushInteractionEntries(before, tableId, triggerType, highlighted, keyPrefix); + afterEntries = after; + } else if (triggerCardFallback) { + // No interaction log — render the trigger card directly. + timelineItems.push({ + key: triggerCardFallback?.key || `${triggerType}-${tableId}`, + type: triggerType, + highlighted, + element: triggerCardFallback, + }); + } + } + // Table card + charts, then reports (output cards, before the summary). + pushTableAndChartItems(tableId, tableCard, tableType, highlighted); + pushReportItems(tableId, highlighted, triggerType); + // The run's closing summary follows the LAST artifact, before any new turn. + if (afterEntries.length > 0) { + pushInteractionEntries(afterEntries, tableId, triggerType, highlighted, `${keyPrefix}-after`); + } + // A new question/explanation on the table follows the summary. + pushTableTextTurns(tableId, highlighted, triggerType); + // Running / clarifying agent state. + pushAgentDraftItems(tableId, triggerType, highlighted); + }; + // Add used (shared) tables at the top // Show the immediate parent as a full table card, with "..." for further ancestors. // On a continuation segment (isSplitThread), suppress the "..." — the @@ -1682,105 +1960,51 @@ let SingleThreadGroupView: FC<{ }); } } - displayedUsedTableIds.forEach((tableId, i) => { + displayedUsedTableIds.forEach((tableId) => { const isHighlighted = highlightedTableIds.includes(tableId); - // On a continuation segment, render the carry-over parent as a - // non-interactive "ghost" so it's clearly an orientation aid, not a - // fresh table — no background color, dashed border, no actions. + // A used parent is a pure SHADOW reference (design-docs/42): the table is + // shown fully — with ALL its attached content (conversation turns, live + // run state) — at its ONE non-ghost card (the source catalog for a source + // table, else the thread that owns it as a fresh table). Here we render + // ONLY the muted ghost card — no turns, no draft. This is what stops a + // fork from repeating the parent's cards / messages / state per column. pushTableAndChartItems( tableId, - _buildTableCard(tableId, { ghost: isSplitThread }), + _buildTableCard(tableId, { ghost: true }), 'table', isHighlighted, ); }); // Interleave triggers and tables for the main thread body - const afterTableMap = new Map(); newTableIds.forEach((tableId, i) => { const triggerPair = newTriggerPairs.find(tp => tp.resultTableId === tableId); - const isHighlighted = highlightedTableIds.includes(tableId); - - // Add trigger card (or interaction log entries) if exists - if (triggerPair) { - const interaction = triggerPair.interaction; - if (interaction && interaction.length > 0) { - const [beforeTable, afterTable] = splitAtLastInstruction(interaction); - pushInteractionEntries(beforeTable, tableId, 'trigger', isHighlighted, 'interaction'); - if (afterTable.length > 0) afterTableMap.set(tableId, afterTable); - } else { - // No interaction log, use trigger card directly - const triggerCard = triggerCards[newTriggerPairs.indexOf(triggerPair)]; - if (triggerCard) { - timelineItems.push({ - key: triggerCard?.key || `woven-trigger-${tableId}`, - type: 'trigger', - highlighted: isHighlighted, - element: triggerCard, - }); - } - } - } - - // Add table card and its charts - pushTableAndChartItems(tableId, tableElementList[i], 'table', isHighlighted); - - // Add report cards anchored to this table. Reports are output cards of - // the run (like charts), so they sit with the other outputs, BEFORE the - // run's closing summary. - pushReportItems(tableId, isHighlighted, 'trigger'); - - // After-table entries (e.g. summary). The run's closing summary is the - // final word and must follow the LAST artifact (table, chart, or - // report), so it is pushed after pushReportItems. - const afterTable = afterTableMap.get(tableId); - if (afterTable && afterTable.length > 0) { - pushInteractionEntries(afterTable, tableId, 'trigger', isHighlighted, 'interaction-after'); - } - - // Running or clarifying agent state - pushAgentDraftItems(tableId, 'trigger', isHighlighted); + pushTableBlock( + tableId, + triggerPair, + tableElementList[i], + triggerPair ? triggerCards[newTriggerPairs.indexOf(triggerPair)] : undefined, + 'table', + 'trigger', + highlightedTableIds.includes(tableId), + 'interaction', + ); }); // Add leaf table components - const leafAfterTableMap = new Map(); - leafTables.forEach((lt, i) => { - let leafTrigger = lt.derive?.trigger; + leafTables.forEach((lt) => { + const leafTrigger = lt.derive?.trigger; const isHL = highlightedTableIds.includes(lt.id); - - if (leafTrigger) { - const interaction = leafTrigger.interaction; - if (interaction && interaction.length > 0) { - const [leafBefore, leafAfter] = splitAtLastInstruction(interaction); - pushInteractionEntries(leafBefore, lt.id, 'leaf-trigger', isHL, 'leaf-interaction'); - if (leafAfter.length > 0) leafAfterTableMap.set(lt.id, leafAfter); - } else { - timelineItems.push({ - key: `leaf-trigger-${lt.id}`, - type: 'leaf-trigger', - highlighted: isHL, - element: _buildTriggerCard(leafTrigger, isHL), - }); - } - } - - pushTableAndChartItems(lt.id, _buildTableCard(lt.id), 'leaf-table', isHL); - - // Add report cards anchored to this leaf table. Reports are output cards - // of the run (like charts), so they sit with the other outputs, BEFORE - // the run's closing summary. - pushReportItems(lt.id, isHL, 'leaf-trigger'); - - // After-table entries (e.g. summary). The run's closing summary is the - // final word and must follow the LAST artifact (table, chart, or - // report), so it is pushed after pushReportItems. - const leafAfterEntries = leafAfterTableMap.get(lt.id); - if (leafAfterEntries && leafAfterEntries.length > 0) { - pushInteractionEntries(leafAfterEntries, lt.id, 'leaf-trigger', isHL, 'leaf-after'); - } - - // Running or clarifying agent state - pushAgentDraftItems(lt.id, 'leaf-trigger', isHL); + pushTableBlock( + lt.id, + leafTrigger, + _buildTableCard(lt.id), + leafTrigger ? _buildTriggerCard(leafTrigger, isHL) : undefined, + 'leaf-table', + 'leaf-trigger', + isHL, + 'leaf-interaction', + ); }); // Timeline rendering helper @@ -1989,9 +2213,15 @@ let SingleThreadGroupView: FC<{ // back into focus. Prefer the latest chart on the associated // table (so users keep seeing the chart they were working on); // fall back to focusing the table itself if no chart exists. + // Also re-opens the pause panel if it was "closed" (dismissed) — + // the thread block is the handle back into the conversation. const clarifyClickHandler = (item.isClarifying && item.tableId) ? () => { const tableId = item.tableId!; + const clarifyDraft = draftNodes.find(d => d.derive?.status === 'clarifying' && d.derive.trigger.tableId === tableId); + if (clarifyDraft) { + window.dispatchEvent(new CustomEvent('df-reopen-pause', { detail: { draftId: clarifyDraft.id } })); + } const chartsForTable = charts.filter(c => c.tableRef === tableId); const lastChart = chartsForTable[chartsForTable.length - 1]; if (lastChart) { @@ -2544,14 +2774,17 @@ function computeSplitExtraLeaves( allTables: DictTable[], chartElements: { tableId: string }[], fittableColumns: number, + textTurnItemsByTable: Map, ): DictTable[] { if (fittableColumns <= 1) return []; const tableById = new Map(allTables.map(t => [t.id, t])); - // Per-trigger item count = 1 (table) + interaction entries + charts. + // Per-trigger item count = 1 (table) + interaction entries + charts + + // text-turn cards (clarify/explain) anchored to the result table. const itemsForTrigger = (resultTableId: string, interaction: InteractionEntry[] | undefined): number => { const charts = chartElements.filter(ce => ce.tableId === resultTableId).length; - return 1 + effectiveEntryCount(interaction) + charts; + const textTurns = textTurnItemsByTable.get(resultTableId) || 0; + return 1 + effectiveEntryCount(interaction) + charts + textTurns; }; // Compute per-thread totals up front so we can pick the budget. @@ -2893,6 +3126,52 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { let generatedReports = useSelector(dfSelectors.getAllGeneratedReports); + // Text turns (clarify/explain) — needed at this level to assign each a + // single "home" thread entry (see the home-assignment block below). + let textTurnsForHome = useSelector((state: DataFormulatorState) => state.textTurns); + // Root TABLE of each turn's conversation (design-docs/42): walk parentNodeId + // until a table (chained follow-ups resolve to their chain's root table). + const textTurnRootByTurn = useMemo(() => { + const tableIds = new Set(tables.map(t => t.id)); + const turnById = new Map(textTurnsForHome.map(tt => [tt.id, tt])); + const rootOf = (tt: TextTurn): string | undefined => { + let cur: TextTurn | undefined = tt; + const seen = new Set(); + while (cur && !seen.has(cur.id)) { + seen.add(cur.id); + const p = cur.parentNodeId; + if (!p) return undefined; + if (tableIds.has(p)) return p; + cur = turnById.get(p); + } + return undefined; + }; + const map = new Map(); + for (const tt of textTurnsForHome) { + const r = rootOf(tt); + if (r) map.set(tt.id, r); + } + return map; + }, [textTurnsForHome, tables]); + // Tables that root a text-turn conversation: branch-split exclusion + home. + const textTurnRootTableIds = useMemo( + () => new Set([...textTurnRootByTurn.values()]), + [textTurnRootByTurn], + ); + // Rendered timeline-item count each table's conversation adds (card + + // optional prompt bubble), keyed by the root table — feeds thread height + + // split budgeting so text-turn cards count as taking vertical space. + const textTurnItemsByTable = useMemo(() => { + const map = new Map(); + for (const tt of textTurnsForHome) { + const key = textTurnRootByTurn.get(tt.id); + if (!key) continue; + const items = 1 + (tt.prompt ? 1 : 0); + map.set(key, (map.get(key) || 0) + items); + } + return map; + }, [textTurnsForHome, textTurnRootByTurn]); + // Derive focusedTableId from focusedId for scroll/highlight logic let focusedTableId = useMemo(() => { if (!focusedId) return undefined; @@ -2905,8 +3184,20 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { const report = generatedReports.find(r => r.id === focusedId.reportId); return report?.triggerTableId; } + if (focusedId.type === 'text') { + // A focused text turn (clarify/explain) highlights its thread-parent + // table (or the table of its source chart) and that table's thread, + // just like focusing a chart (design-docs/41/42). + const turn = textTurnsForHome.find(tt => tt.id === focusedId.textId); + if (!turn) return undefined; + if (turn.sourceChartId) { + const chart = charts.find(c => c.id === turn.sourceChartId); + if (chart?.tableRef) return chart.tableRef; + } + return textTurnRootByTurn.get(turn.id); + } return undefined; - }, [focusedId, charts, generatedReports]); + }, [focusedId, charts, generatedReports, textTurnsForHome, textTurnRootByTurn]); let chartSynthesisInProgress = useSelector((state: DataFormulatorState) => state.chartSynthesisInProgress); @@ -3131,6 +3422,9 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { // anchors are considered leaf tables to simplify the view let isLeafTable = (table: DictTable) => { + // A table with no (non-anchored) derivations is a leaf. Conversation- + // produced tables are NORMAL tables now (design-docs/42): they fork into + // their own column via the standard leaf partition, so no special case. let children = tables.filter(t => t.derive?.trigger.tableId == table.id); if (children.length == 0 || children.every(t => t.anchored)) { return true; @@ -3165,11 +3459,19 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { const computedExtras = fittableColumns <= 1 ? [] : computeSplitExtraLeaves( - leafTables, tables, chartElements, fittableColumns, + leafTables, tables, chartElements, fittableColumns, textTurnItemsByTable, ); // Avoid duplicating tables that are already leaves (e.g. anchored mids). + // Also never split at a table that carries a terminal text turn + // (clarify/explain with no result table): promoting it as a segment + // endpoint would strand its explanation in a separate thread column, + // divorced from the derivations that continue from the same table + // (design-docs/41). Keeping it un-promoted glues the explanation to the + // table's outgoing derivation flow in one continuous thread. const existingLeafIds = new Set(leafTables.map(t => t.id)); - const extraLeaves: DictTable[] = computedExtras.filter(t => !existingLeafIds.has(t.id)); + const extraLeaves: DictTable[] = computedExtras.filter( + t => !existingLeafIds.has(t.id) && !textTurnRootTableIds.has(t.id), + ); if (extraLeaves.length > 0) { leafTables = [...leafTables, ...extraLeaves]; } @@ -3250,7 +3552,9 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { // Collect all tables (including derived ones) for the workspace panel. let baseTables = tables; - // Threaded tables: leaf tables that have a derivation chain + // Threaded tables: leaf tables that have a derivation chain. A conversation- + // produced table is a normal derived leaf, so it threads (forks) here without + // any special case (design-docs/42). let threadedTables = leafTables.filter(lt => { const triggers = getTriggers(lt, tables); return triggers.length + 1 > 1; @@ -3335,6 +3639,10 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { let chartCount = newTableIds.reduce((sum, tid) => sum + chartElements.filter(ce => ce.tableId === tid).length, 0); let entryCount = newTriggerPairs.reduce((sum, tp) => sum + (tp.interaction?.length || 1), 0); entryCount += lt.derive?.trigger?.interaction?.length || 1; + // Text-turn cards (clarify/explain) anchored to any table in this + // thread also occupy vertical space — count them so tall conversations + // widen/split correctly. + entryCount += [...threadTableIds].reduce((sum, id) => sum + (textTurnItemsByTable.get(id) || 0), 0); let totalTables = newTableIds.length + 1; threadTableIds.forEach(id => claimedTableIds.add(id)); @@ -3385,7 +3693,12 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { } } - // Pick the best column layout: dynamically based on container width. + // (design-docs/42) No per-turn home assignment anymore: a table's attached + // content (conversation turns + live run state) renders at its single + // NON-GHOST card (source catalog for a source table, else the owning thread; + // see pushTextTurnSubtree / the ghost used-parent loop). Threads/columns come + // purely from table leaves via the standard split rules. + // Use the same stable viewportHeight (derived from the outer wrapper) for // packing as we do for split decisions — so the column count doesn't shift // when the chatbox grows during clarification/explanation. diff --git a/src/views/InteractionEntryCard.tsx b/src/views/InteractionEntryCard.tsx index 0f7d3ae2..c62d4ca4 100644 --- a/src/views/InteractionEntryCard.tsx +++ b/src/views/InteractionEntryCard.tsx @@ -4,6 +4,7 @@ import React, { memo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import Markdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; import { Box, Collapse, Typography, useTheme } from '@mui/material'; import { alpha } from '@mui/material/styles'; import PersonIcon from '@mui/icons-material/Person'; @@ -17,6 +18,7 @@ import CheckIcon from '@mui/icons-material/Check'; import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; import WarningAmberIcon from '@mui/icons-material/WarningAmber'; import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; +import AttachFileIcon from '@mui/icons-material/AttachFile'; import { InteractionEntry } from '../components/ComponentType'; import { AgentIcon } from '../icons'; import { radius, borderColor } from '../app/tokens'; @@ -132,13 +134,21 @@ export const PlanStepsView: React.FC<{ }; /** Compact Markdown for summary entries — inherits parent font-size (10px). */ -export const CompactMarkdown: React.FC<{ content: string; color: string }> = ({ content, color }) => ( +export const CompactMarkdown: React.FC<{ content: string; color: string }> = ({ content, color }) => { + const theme = useTheme(); + return ( :first-child': { mt: 0 }, '& > :last-child': { mb: 0 }, }}> ( @@ -164,19 +174,43 @@ export const CompactMarkdown: React.FC<{ content: string; color: string }> = ({ ), code: ({ children }) => ( {children} ), pre: ({ children }) => <>{children}, + table: ({ children }) => ( + + + {children} + + + ), + th: ({ children }) => ( + + {children} + + ), + td: ({ children }) => ( + + {children} + + ), }} > {content} -); + ); +}; /** Render text with `**field**` markers as styled spans. The marker is * rendered as a flat "highlighter underline" — a thin colored bar sitting @@ -257,9 +291,26 @@ export const InteractionEntryCard: React.FC = memo(({ ...(highlighted ? { borderLeft: `2px solid ${palette.main}` } : {}), ...clickSx, }}> - + {renderFieldHighlights(text, palette.main)} + {entry.attachments && entry.attachments.length > 0 && ( + + {entry.attachments.map((name, i) => ( + + + {name} + + ))} + + )} ); } @@ -318,7 +369,21 @@ export const InteractionEntryCard: React.FC = memo(({ color = theme.palette.text.secondary; } - const hasPlan = !!entry.plan && entry.plan !== displayText; + // Plan (thinking) lines: split, then drop the redundant "creating + // chart…" step (it just duplicates the instruction text). A plan whose + // ONLY content is that filtered step has nothing to show — so `hasPlan` + // is false and no thinking section / divider renders above the text. + const planLinesVisible = (() => { + if (!entry.plan || entry.plan === displayText) return [] as string[]; + const raw = (entry.plan.includes('\x1E') ? entry.plan.split('\x1E') : entry.plan.split('\n')) + .filter(l => l.trim()); + return raw.filter(l => { + const stripped = l.startsWith('✓') ? l.slice(2) : l; + const lbl = stripped.trim().toLowerCase(); + return !(lbl.startsWith('creating chart') || lbl.startsWith('图表')); + }); + })(); + const hasPlan = planLinesVisible.length > 0; // Active clarify/explain entries are read in the ClarificationPanel // at the bottom (the outer timeline row already refocuses there on @@ -443,14 +508,11 @@ export const InteractionEntryCard: React.FC = memo(({ onClick={() => isCollapsible && setExpanded(!expanded)} > - {hasPlan && (() => { - const planLines = (entry.plan!.includes('\x1E') ? entry.plan!.split('\x1E') : entry.plan!.split('\n')).filter(l => l.trim()); - return ( - - - - ); - })()} + {hasPlan && ( + + + + )} {hasPlan && } {collapsedLabel && ( = memo(({ fontSize: '11px', color, py: '1px', + wordBreak: 'break-word', + overflowWrap: 'anywhere', ...((forceClampText || (canClampText && !expanded)) ? { display: '-webkit-box', WebkitLineClamp: TEXT_CLAMP_LINES, @@ -537,6 +601,9 @@ export const InteractionEntryCard: React.FC = memo(({ export interface ResolvedConversationCardProps { pairs: { agentEntry: InteractionEntry; userEntry: InteractionEntry }[]; highlighted?: boolean; + /** Source table whose interaction holds these entries — lets the re-opened + * explanation popup delete this block from the thread. */ + sourceTableId?: string; } /** Render one or more resolved clarify/explain/suggest_data_search @@ -550,9 +617,8 @@ export interface ResolvedConversationCardProps { * hinted "💬 conversation happened here" marker that stays openable * for context. */ -export const ResolvedConversationCard: React.FC = memo(({ pairs }) => { +export const ResolvedConversationCard: React.FC = memo(({ pairs, sourceTableId }) => { const theme = useTheme(); - const { t } = useTranslation(); const [expanded, setExpanded] = useState(false); if (pairs.length === 0) return null; @@ -560,61 +626,84 @@ export const ResolvedConversationCard: React.FC = // Preview uses the LAST user reply (most recent resolution); fall back // to the last agent question if that reply is empty. const lastPair = pairs[pairs.length - 1]; - const lastUserText = stripFieldMarkers(lastPair.userEntry.displayContent || lastPair.userEntry.content).replace(/\s+/g, ' ').trim(); - const lastAgentText = stripFieldMarkers(lastPair.agentEntry.displayContent || lastPair.agentEntry.content).replace(/\s+/g, ' ').trim(); - const previewText = lastUserText || lastAgentText; + // Compact card preview: the agent's message (question / answer) plus the + // user's follow-up reply, shown as `↳ …`. + const agentPreview = stripFieldMarkers(lastPair.agentEntry.displayContent || lastPair.agentEntry.content || '') + .replace(/[#*`>|]/g, ' ').replace(/\s+/g, ' ').trim(); + const followup = stripFieldMarkers(lastPair.userEntry.displayContent || lastPair.userEntry.content || '') + .replace(/\s+/g, ' ').trim(); + + // An explanation exchange (agent gave an answer) re-opens the full + // read-only ExplanationPanel popup on click — which renders its markdown + // (tables, code) properly — rather than the plain inline expand used for + // clarify/delegate back-and-forth. Bridged to SimpleChartRecBox via a + // window CustomEvent (same pattern as `df-replay-workflow`) to avoid + // growing the shared redux slice. + const isExplanation = pairs.every(p => p.agentEntry.role === 'explain'); + const handleCardClick = () => { + if (isExplanation) { + const md = lastPair.agentEntry.content || lastPair.agentEntry.displayContent || ''; + if (md.trim()) { + // Timestamps of every entry in this conversation so the popup's + // Delete can remove the whole block from the source table. + const timestamps = pairs.flatMap(p => [p.agentEntry.timestamp, p.userEntry.timestamp]) + .filter((t): t is number => typeof t === 'number'); + window.dispatchEvent(new CustomEvent('df-view-explanation', { + detail: { content: md, sourceTableId, timestamps }, + })); + } + } else { + setExpanded(v => !v); + } + }; - const dim = theme.palette.text.secondary; const customPalette = theme.palette.custom; const turnCount = pairs.length; return ( - setExpanded(v => !v)} - sx={{ - cursor: 'pointer', - py: '2px', - px: '4px', - borderRadius: '4px', - '&:hover': { backgroundColor: 'rgba(0,0,0,0.03)' }, - }} - > + {!expanded ? ( + // Simple card: agent message preview + ↳ user reply. Same look + // for clarify / explain / delegate (primary-tinted). Explain + // clicks re-open the full popup; the others expand inline below. - {turnCount > 1 && ( - + {turnCount > 1 && ( + + ×{turnCount} + + )} + + {agentPreview} + + + )} + {followup && ( + - ×{turnCount} + ↳ {followup} )} - - {previewText} - ) : ( diff --git a/src/views/LogViewerDialog.tsx b/src/views/LogViewerDialog.tsx new file mode 100644 index 00000000..bb357ddf --- /dev/null +++ b/src/views/LogViewerDialog.tsx @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * LogViewerDialog — view and download the persistent server log. + * + * Only mounted when the server reports local single-user mode + * (`serverConfig.IS_LOCAL_MODE`). In hosted deployments the log endpoints + * return ACCESS_DENIED and this button is never rendered. + * + * The log file lives at `/logs/data_formulator.log` + * and captures all server + Python-execution output — the artifact a user + * can send when reporting an issue. + */ + +import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; +import { + Box, + Button, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + IconButton, + Tooltip, + Typography, +} from '@mui/material'; +import TerminalOutlinedIcon from '@mui/icons-material/TerminalOutlined'; +import RefreshIcon from '@mui/icons-material/Refresh'; +import DownloadIcon from '@mui/icons-material/Download'; +import { useTranslation } from 'react-i18next'; + +import { getUrls } from '../app/utils'; +import { apiRequest } from '../app/apiClient'; + +const TAIL_LINES = 2000; + +interface LogTailResponse { + path: string | null; + exists: boolean; + lines?: number; + content: string; +} + +export const LogViewerDialog: FC = () => { + const { t } = useTranslation(); + const [open, setOpen] = useState(false); + const [loading, setLoading] = useState(false); + const [content, setContent] = useState(''); + const [path, setPath] = useState(null); + const [error, setError] = useState(null); + const preRef = useRef(null); + + const fetchLogs = useCallback(async () => { + setLoading(true); + setError(null); + try { + const { data } = await apiRequest( + `${getUrls().LOGS_TAIL}?lines=${TAIL_LINES}`, + ); + setContent(data.content || ''); + setPath(data.path ?? null); + } catch (e: any) { + setError(e?.message || 'Failed to load logs'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + if (open) { + fetchLogs(); + } + }, [open, fetchLogs]); + + // Auto-scroll to the newest line once content renders. + useEffect(() => { + if (open && preRef.current) { + preRef.current.scrollTop = preRef.current.scrollHeight; + } + }, [content, open]); + + const handleDownload = () => { + // Direct navigation triggers the browser download (attachment header). + window.open(getUrls().LOGS_DOWNLOAD, '_blank'); + }; + + return ( + <> + + setOpen(true)} + sx={{ + p: 0.5, + color: 'text.secondary', + '&:hover': { color: 'text.primary', backgroundColor: 'rgba(0, 0, 0, 0.04)' }, + }} + aria-label={t('logs.viewLogs', { defaultValue: 'View server logs' })} + > + + + + setOpen(false)} maxWidth="lg" fullWidth> + + + {t('logs.title', { defaultValue: 'Server Logs' })} + + + + + + + + + + + + + + + + + + {path && ( + + {path} + + )} + {loading && ( + + + + )} + {!loading && error && ( + + {error} + + )} + {!loading && !error && ( + + {content || t('logs.empty', { defaultValue: 'Log file is empty.' })} + + )} + + + + + + + ); +}; diff --git a/src/views/ModelSelectionDialog.tsx b/src/views/ModelSelectionDialog.tsx index d067ac16..051f5112 100644 --- a/src/views/ModelSelectionDialog.tsx +++ b/src/views/ModelSelectionDialog.tsx @@ -552,7 +552,22 @@ export const ModelSelectionButton: React.FC<{}> = ({ }) => { return <> -