From 63715b09ce348e494ee139158cd9b7aaa0ef92a4 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Tue, 14 Jul 2026 01:24:03 -0700 Subject: [PATCH 01/17] feat: persistent server logs + local log viewer, app-bar refinement, 1.5x default stretch - Persistent rotating log file under /logs (sanitized); local-mode-only /api/logs endpoints + toolbar log viewer. - Log AI-generated code execution failures (explore/visualize/data-loading) at the agent layer so bugs surface in the log with detail. - Refine top app bar: muted/slim icons (outlined settings, terminal log icon), size-aligned buttons matching the left-nav standard. - Lower default chart maxStretchFactor 2.0 -> 1.5. --- .../agents/agent_data_loading_chat.py | 7 +- py-src/data_formulator/analyst/agent.py | 11 +- py-src/data_formulator/app.py | 94 ++++++++- py-src/data_formulator/routes/logs.py | 116 +++++++++++ src/app/App.tsx | 95 +++++---- src/app/dfSlice.tsx | 4 +- src/app/utils.tsx | 5 + src/i18n/locales/en/common.json | 7 + src/i18n/locales/zh/common.json | 7 + src/views/LogViewerDialog.tsx | 183 ++++++++++++++++++ src/views/ModelSelectionDialog.tsx | 17 +- 11 files changed, 505 insertions(+), 41 deletions(-) create mode 100644 py-src/data_formulator/routes/logs.py create mode 100644 src/views/LogViewerDialog.tsx diff --git a/py-src/data_formulator/agents/agent_data_loading_chat.py b/py-src/data_formulator/agents/agent_data_loading_chat.py index 7f5d36d4..e31fda62 100644 --- a/py-src/data_formulator/agents/agent_data_loading_chat.py +++ b/py-src/data_formulator/agents/agent_data_loading_chat.py @@ -1268,9 +1268,14 @@ def _tool_execute_python(self, args): return response else: + err = raw.get("error_message", raw.get("content", "Unknown error")) + logger.warning( + "execute_python code failed: %s\n--- code ---\n%s", + err, code[:2000], + ) return { "stdout": "", - "error": raw.get("error_message", raw.get("content", "Unknown error")), + "error": err, } except Exception as e: diff --git a/py-src/data_formulator/analyst/agent.py b/py-src/data_formulator/analyst/agent.py index 078fb610..e4d85be1 100644 --- a/py-src/data_formulator/analyst/agent.py +++ b/py-src/data_formulator/analyst/agent.py @@ -973,9 +973,14 @@ def _run_explore_code( stdout = stdout[:8000] + "\n... (truncated)" return {"status": "ok", "stdout": stdout} else: + err = raw.get("error_message", raw.get("content", "Unknown error")) + logger.warning( + "[AnalystAgent] explore code failed: %s\n--- code ---\n%s", + err, code[:2000], + ) return { "status": "error", - "error": raw.get("error_message", raw.get("content", "Unknown error")), + "error": err, "stdout": "", } except Exception as e: @@ -1019,6 +1024,10 @@ def _run_visualize_code( if execution_result['status'] != 'ok': error_message = execution_result.get('content', 'Unknown error') + logger.warning( + "[AnalystAgent] visualize code failed: %s\n--- code ---\n%s", + error_message, code[:2000], + ) return {"status": "error", "error_message": str(error_message)} full_df = execution_result['content'] diff --git a/py-src/data_formulator/app.py b/py-src/data_formulator/app.py index 64bf5158..dde1a5c0 100644 --- a/py-src/data_formulator/app.py +++ b/py-src/data_formulator/app.py @@ -123,6 +123,85 @@ def default(self, obj): # Get logger for this module (logging config moved to run_app function) logger = logging.getLogger(__name__) +_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +# Marker attribute so we never attach the persistent file handler twice +# (configure_logging runs early; run_app re-checks once --data-dir is known). +_FILE_HANDLER_MARKER = '_df_persistent_file_handler' + + +def _resolve_data_home() -> Path: + """Resolve the Data Formulator home directory for logging. + + Mirrors ``get_data_formulator_home()`` but is safe to call outside an + application context (logging is configured before the app runs). + Order: CLI_ARGS['data_dir'] > DATA_FORMULATOR_HOME env > ~/.data_formulator. + """ + data_dir = app.config.get('CLI_ARGS', {}).get('data_dir') + if data_dir: + return Path(data_dir) + env_home = os.getenv('DATA_FORMULATOR_HOME') + if env_home: + return Path(env_home) + return Path.home() / '.data_formulator' + + +def configure_file_logging() -> None: + """Attach a rotating file handler that persists all logs under + ``/logs/data_formulator.log``. + + This is the artifact users can send when reporting problems. Output is + passed through :class:`SensitiveDataFilter` so API keys / tokens are + redacted. The handler is idempotent — if the resolved path changes + (e.g. ``--data-dir`` supplied after early configuration) the old handler + is replaced. + """ + from logging.handlers import RotatingFileHandler + from data_formulator.security.log_sanitizer import SensitiveDataFilter + + try: + log_dir = _resolve_data_home() / 'logs' + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / 'data_formulator.log' + except Exception as exc: # pragma: no cover - filesystem edge cases + logging.getLogger('data_formulator').warning( + "Could not set up persistent file logging: %s", exc, + ) + return + + root = logging.getLogger() + + # Skip if an identical handler is already attached; otherwise drop any + # previous persistent handler so we don't duplicate output. + for h in list(root.handlers): + if getattr(h, _FILE_HANDLER_MARKER, False): + if getattr(h, 'baseFilename', None) == str(log_path.resolve()): + app.config['LOG_FILE_PATH'] = str(log_path) + return + root.removeHandler(h) + try: + h.close() + except Exception: + pass + + handler = RotatingFileHandler( + log_path, maxBytes=10 * 1024 * 1024, backupCount=5, encoding='utf-8', + ) + setattr(handler, _FILE_HANDLER_MARKER, True) + handler.setLevel(logging.DEBUG) + handler.setFormatter(logging.Formatter(_LOG_FORMAT)) + handler.addFilter(SensitiveDataFilter()) + root.addHandler(handler) + + # Mirror onto the Flask app logger too (its handlers are managed separately). + if not any(getattr(h, _FILE_HANDLER_MARKER, False) for h in app.logger.handlers): + app.logger.addHandler(handler) + + app.config['LOG_FILE_PATH'] = str(log_path) + logging.getLogger('data_formulator').info( + "Persistent logs: %s", log_path, + ) + + def configure_logging(): """Configure logging for the Flask application.""" log_level_str = os.getenv("LOG_LEVEL", "INFO").strip().upper() @@ -135,7 +214,7 @@ def configure_logging(): logging.basicConfig( level=logging.WARNING, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + format=_LOG_FORMAT, handlers=[handler], ) @@ -149,6 +228,9 @@ def configure_logging(): for h in logging.getLogger().handlers: app.logger.addHandler(h) + # Persist everything to a file under the DF home so users can send logs. + configure_file_logging() + logging.getLogger('data_formulator').info( "Log level: %s (sanitize=%s)", log_level_str, os.getenv("LOG_SANITIZE", "true"), @@ -188,12 +270,16 @@ def _register_blueprints(): # Import demo stream routes from data_formulator.routes.demo_stream import demo_stream_bp, limiter as demo_stream_limiter demo_stream_limiter.init_app(app) - + + # Import server-log inspection routes (local-mode gated) + from data_formulator.routes.logs import logs_bp + # Register blueprints app.register_blueprint(tables_bp) app.register_blueprint(agent_bp) app.register_blueprint(session_bp) app.register_blueprint(demo_stream_bp) + app.register_blueprint(logs_bp) # Initialise pluggable authentication (reads AUTH_PROVIDER env var) from data_formulator.auth.identity import init_auth, get_active_provider @@ -446,6 +532,10 @@ def run_app(): ], } + # Now that --data-dir is applied, ensure the persistent log file lives + # under the resolved DF home (re-points if it differs from the default). + configure_file_logging() + # Register blueprints (this is where heavy imports happen) _register_blueprints() diff --git a/py-src/data_formulator/routes/logs.py b/py-src/data_formulator/routes/logs.py new file mode 100644 index 00000000..5377c5e4 --- /dev/null +++ b/py-src/data_formulator/routes/logs.py @@ -0,0 +1,116 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +""" +Server log inspection routes. + +Data Formulator persists all server + Python-execution logs to a rotating +file under ``/logs/data_formulator.log`` (configured in +``app.configure_file_logging``). This is the artifact a user can send when +reporting a problem. + +Access policy — logs are **server-side only**: + +* In **local single-user mode** (``is_local_mode()`` is true) the user *is* + the server operator, so these endpoints expose the log to the UI (view / + tail / download). +* In any **hosted / multi-user** deployment these endpoints return + ``ACCESS_DENIED``. The operator reads the file directly on the server host; + end users never see server logs. + +Routes: + GET /api/logs/info — metadata: path, size, existence, local-mode flag + GET /api/logs/tail — last N lines of the current log file + GET /api/logs/download — download the current log file +""" + +import logging +import os +from collections import deque + +from flask import Blueprint, current_app, request, send_file + +from data_formulator.auth.identity import is_local_mode +from data_formulator.error_handler import json_ok +from data_formulator.errors import AppError, ErrorCode + +logger = logging.getLogger(__name__) + +logs_bp = Blueprint("logs", __name__, url_prefix="/api/logs") + +# Hard cap so a huge log can never blow up memory / the response. +_MAX_TAIL_LINES = 5000 +_DEFAULT_TAIL_LINES = 500 + + +def _require_local_mode() -> None: + """Reject the request unless running in local single-user mode.""" + if not is_local_mode(): + raise AppError( + ErrorCode.ACCESS_DENIED, + "Server logs are only viewable in local mode.", + ) + + +def _log_path() -> str | None: + """Resolve the active log file path (set by configure_file_logging).""" + return current_app.config.get("LOG_FILE_PATH") + + +@logs_bp.route("/info", methods=["GET"]) +def logs_info(): + """Return log file metadata (local mode only).""" + _require_local_mode() + path = _log_path() + exists = bool(path) and os.path.isfile(path) + size = os.path.getsize(path) if exists else 0 + return json_ok({ + "path": path, + "exists": exists, + "size": size, + }) + + +@logs_bp.route("/tail", methods=["GET"]) +def logs_tail(): + """Return the last N lines of the current log file (local mode only).""" + _require_local_mode() + path = _log_path() + if not path or not os.path.isfile(path): + return json_ok({"path": path, "exists": False, "lines": [], "content": ""}) + + try: + lines = int(request.args.get("lines", _DEFAULT_TAIL_LINES)) + except (TypeError, ValueError): + lines = _DEFAULT_TAIL_LINES + lines = max(1, min(lines, _MAX_TAIL_LINES)) + + # deque with maxlen keeps memory bounded regardless of file size. + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + tail = deque(fh, maxlen=lines) + except OSError as exc: + raise AppError(ErrorCode.VALIDATION_ERROR, f"Could not read log file: {exc}") + + content = "".join(tail) + return json_ok({ + "path": path, + "exists": True, + "lines": len(tail), + "content": content, + }) + + +@logs_bp.route("/download", methods=["GET"]) +def logs_download(): + """Download the current log file as an attachment (local mode only).""" + _require_local_mode() + path = _log_path() + if not path or not os.path.isfile(path): + raise AppError(ErrorCode.VALIDATION_ERROR, "No log file available.") + return send_file( + path, + mimetype="text/plain", + as_attachment=True, + download_name="data_formulator.log", + ) diff --git a/src/app/App.tsx b/src/app/App.tsx index c97f4bd6..4dacdabe 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -67,7 +67,7 @@ import { useWorkspaceAutoName } from './useWorkspaceAutoName'; import GridViewIcon from '@mui/icons-material/GridView'; import ViewSidebarIcon from '@mui/icons-material/ViewSidebar'; -import SettingsIcon from '@mui/icons-material/Settings'; +import SettingsOutlinedIcon from '@mui/icons-material/SettingsOutlined'; import { createBrowserRouter, Link as RouterLink, @@ -84,6 +84,7 @@ import { AppDispatch } from './store'; import dfLogo from '../assets/df-logo.png'; import { AnvilLoader } from '../components/AnvilLoader'; import { ModelSelectionButton } from '../views/ModelSelectionDialog'; +import { LogViewerDialog } from '../views/LogViewerDialog'; import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'; import UploadFileIcon from '@mui/icons-material/UploadFile'; import DownloadIcon from '@mui/icons-material/Download'; @@ -206,12 +207,16 @@ const LanguageSwitcher: React.FC = () => { sx={{ height: '28px', my: 'auto', - mr: 1, '& .MuiToggleButton-root': { textTransform: 'none', fontSize: '12px', py: 0, minWidth: '40px', + color: 'text.secondary', + borderColor: 'divider', + '&.Mui-selected': { + color: 'text.primary', + }, }, }} > @@ -465,8 +470,18 @@ const ExitSessionButton: React.FC = () => { size="small" variant="text" onClick={handleExit} - startIcon={} - sx={{ textTransform: 'none', color: 'text.secondary' }} + startIcon={} + sx={{ + textTransform: 'none', + fontSize: '13px', + fontWeight: 400, + px: 1.5, + py: 0.5, + minWidth: 'auto', + lineHeight: 1.5, + color: 'text.secondary', + '&:hover': { color: 'text.primary', backgroundColor: 'rgba(0, 0, 0, 0.04)' }, + }} > {t('workspace.exit', { defaultValue: 'Exit' })} @@ -487,7 +502,7 @@ const ConfigDialog: React.FC = () => { const [formulateTimeoutSeconds, setFormulateTimeoutSeconds] = useState(config.formulateTimeoutSeconds ?? 180); const [defaultChartWidth, setDefaultChartWidth] = useState(config.defaultChartWidth ?? 300); const [defaultChartHeight, setDefaultChartHeight] = useState(config.defaultChartHeight ?? 300); - const [maxStretchFactor, setMaxStretchFactor] = useState(config.maxStretchFactor ?? 2.0); + const [maxStretchFactor, setMaxStretchFactor] = useState(config.maxStretchFactor ?? 1.5); const [frontendRowLimit, setFrontendRowLimit] = useState(config.frontendRowLimit ?? rowLimitDefault); const [paletteKey, setPaletteKey] = useState( (config.paletteKey && palettes[config.paletteKey]) ? config.paletteKey : defaultPaletteKey @@ -502,9 +517,20 @@ const ConfigDialog: React.FC = () => { return ( <> - + + setOpen(true)} + aria-label={t('app.settings')} + sx={{ + p: 0.5, + color: 'text.secondary', + '&:hover': { color: 'text.primary', backgroundColor: 'rgba(0, 0, 0, 0.04)' }, + }} + > + + + setOpen(false)} open={open}> {t('app.settings')} @@ -745,9 +771,9 @@ const AppShell: FC = () => { const viewMode = useSelector((state: DataFormulatorState) => state.viewMode); const tables = useSelector((state: DataFormulatorState) => state.tables); const activeWorkspace = useSelector((state: DataFormulatorState) => state.activeWorkspace); + const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); - useEffect(() => { - const authError = searchParams.get('auth_error'); + useEffect(() => { const authError = searchParams.get('auth_error'); if (!authError) return; const i18nKey = AUTH_ERROR_MESSAGES[authError] || 'auth.ssoErrorGeneric'; dispatch(dfActions.addMessages({ @@ -841,14 +867,34 @@ const AppShell: FC = () => { )} {isAppPage && ( - - - - + + + + + + {serverConfig.IS_LOCAL_MODE && } + + + + + + {activeWorkspace && ( <> - + )} @@ -927,25 +973,6 @@ const AppShell: FC = () => { )} - {isAppPage && ( - - - - )} diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 581ea618..114fd0ac 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -113,7 +113,7 @@ export interface ClientConfig { formulateTimeoutSeconds: number; defaultChartWidth: number; defaultChartHeight: number; - maxStretchFactor: number; // max per-axis stretch multiplier for chart sizing (default 2.0) + maxStretchFactor: number; // max per-axis stretch multiplier for chart sizing (default 1.5) frontendRowLimit: number; // max rows to keep in browser when loading locally (non-virtual) paletteKey: string; // active color palette key from tokens.ts miniMode: boolean; // when true, run the single-turn MiniAnalystAgent (for small/local models) @@ -333,7 +333,7 @@ const initialState: DataFormulatorState = { formulateTimeoutSeconds: 180, defaultChartWidth: 400, defaultChartHeight: 300, - maxStretchFactor: 2.0, + maxStretchFactor: 1.5, frontendRowLimit: DEFAULT_ROW_LIMIT, paletteKey: 'fluent', miniMode: false, diff --git a/src/app/utils.tsx b/src/app/utils.tsx index b288a27f..cc9c73b0 100644 --- a/src/app/utils.tsx +++ b/src/app/utils.tsx @@ -67,6 +67,11 @@ export function getUrls() { // Refresh data endpoint REFRESH_DERIVED_DATA: `/api/agent/refresh-derived-data`, + // Server logs (local mode only) + LOGS_INFO: `/api/logs/info`, + LOGS_TAIL: `/api/logs/tail`, + LOGS_DOWNLOAD: `/api/logs/download`, + // Session management SESSION_SAVE: `/api/sessions/save`, SESSION_LIST: `/api/sessions/list`, diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 44c8109c..4f0a5219 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -47,6 +47,13 @@ "data": "Data", "microsoftResearch": "Microsoft Research" }, + "logs": { + "title": "Server Logs", + "viewLogs": "View server logs", + "refresh": "Refresh", + "download": "Download full log", + "empty": "Log file is empty." + }, "session": { "exportSession": "export session", "importSession": "import session", diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index a718906d..47e1de8c 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -47,6 +47,13 @@ "data": "数据", "microsoftResearch": "微软研究院" }, + "logs": { + "title": "服务器日志", + "viewLogs": "查看服务器日志", + "refresh": "刷新", + "download": "下载完整日志", + "empty": "日志文件为空。" + }, "session": { "exportSession": "导出会话", "importSession": "导入会话", diff --git a/src/views/LogViewerDialog.tsx b/src/views/LogViewerDialog.tsx new file mode 100644 index 00000000..bb357ddf --- /dev/null +++ b/src/views/LogViewerDialog.tsx @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * LogViewerDialog — view and download the persistent server log. + * + * Only mounted when the server reports local single-user mode + * (`serverConfig.IS_LOCAL_MODE`). In hosted deployments the log endpoints + * return ACCESS_DENIED and this button is never rendered. + * + * The log file lives at `/logs/data_formulator.log` + * and captures all server + Python-execution output — the artifact a user + * can send when reporting an issue. + */ + +import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; +import { + Box, + Button, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + IconButton, + Tooltip, + Typography, +} from '@mui/material'; +import TerminalOutlinedIcon from '@mui/icons-material/TerminalOutlined'; +import RefreshIcon from '@mui/icons-material/Refresh'; +import DownloadIcon from '@mui/icons-material/Download'; +import { useTranslation } from 'react-i18next'; + +import { getUrls } from '../app/utils'; +import { apiRequest } from '../app/apiClient'; + +const TAIL_LINES = 2000; + +interface LogTailResponse { + path: string | null; + exists: boolean; + lines?: number; + content: string; +} + +export const LogViewerDialog: FC = () => { + const { t } = useTranslation(); + const [open, setOpen] = useState(false); + const [loading, setLoading] = useState(false); + const [content, setContent] = useState(''); + const [path, setPath] = useState(null); + const [error, setError] = useState(null); + const preRef = useRef(null); + + const fetchLogs = useCallback(async () => { + setLoading(true); + setError(null); + try { + const { data } = await apiRequest( + `${getUrls().LOGS_TAIL}?lines=${TAIL_LINES}`, + ); + setContent(data.content || ''); + setPath(data.path ?? null); + } catch (e: any) { + setError(e?.message || 'Failed to load logs'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + if (open) { + fetchLogs(); + } + }, [open, fetchLogs]); + + // Auto-scroll to the newest line once content renders. + useEffect(() => { + if (open && preRef.current) { + preRef.current.scrollTop = preRef.current.scrollHeight; + } + }, [content, open]); + + const handleDownload = () => { + // Direct navigation triggers the browser download (attachment header). + window.open(getUrls().LOGS_DOWNLOAD, '_blank'); + }; + + return ( + <> + + setOpen(true)} + sx={{ + p: 0.5, + color: 'text.secondary', + '&:hover': { color: 'text.primary', backgroundColor: 'rgba(0, 0, 0, 0.04)' }, + }} + aria-label={t('logs.viewLogs', { defaultValue: 'View server logs' })} + > + + + + setOpen(false)} maxWidth="lg" fullWidth> + + + {t('logs.title', { defaultValue: 'Server Logs' })} + + + + + + + + + + + + + + + + + + {path && ( + + {path} + + )} + {loading && ( + + + + )} + {!loading && error && ( + + {error} + + )} + {!loading && !error && ( + + {content || t('logs.empty', { defaultValue: 'Log file is empty.' })} + + )} + + + + + + + ); +}; diff --git a/src/views/ModelSelectionDialog.tsx b/src/views/ModelSelectionDialog.tsx index d067ac16..051f5112 100644 --- a/src/views/ModelSelectionDialog.tsx +++ b/src/views/ModelSelectionDialog.tsx @@ -552,7 +552,22 @@ export const ModelSelectionButton: React.FC<{}> = ({ }) => { return <> -