Dev

README.md

@@ -43,7 +43,7 @@ https://github.com/user-attachments/assets/8e4f8a08-6423-4227-a1f7-559e0126ce31
 
 ## News 🔥🔥🔥
 
-[07-23-2026] **Data Formulator 0.8 alpha** (a1–a3, latest: 0.8.0a3) includes:
+[07-23-2026] **Data Formulator 0.8 alpha** (a1–a4, latest: 0.8.0a4) includes:
 
 - **Conversational database loading.** Agents can discover relevant tables, propose filters, preview results, and revise a loading plan through conversation before importing data.
 - **Unified Data Thread.** Questions, clarifications, explanations, tables, and charts share one conversation history, with branching from earlier steps into new questions, calculated columns, or visualizations.
@@ -53,7 +53,7 @@ https://github.com/user-attachments/assets/8e4f8a08-6423-4227-a1f7-559e0126ce31
 - **Microsoft authentication for enterprise connectors.** SQL Server supports passwordless Microsoft Entra ID authentication through `az login`, including an in-app flow for local deployments. Kusto supports delegated Microsoft sign-in alongside Azure default identity and service principal authentication.
 - **Connector setup and diagnostics.** Connection forms separate connection, scope, and source-specific authentication options. Persistent server logs and an in-app log viewer help diagnose failures.
 
-> Preview with `pip install --pre data_formulator==0.8.0a3` or `uvx data_formulator@0.8.0a3`.
+> Preview with `pip install --pre data_formulator==0.8.0a4` or `uvx data_formulator@0.8.0a4`.
 
 > Install the latest stable release (0.7) with `pip install data_formulator` or run instantly with `uvx data_formulator`.
 

docs/dev-guides/14-model-capability-runtime-degradation.md

@@ -9,7 +9,7 @@
 
 1. **唯一调用路径** — 所有 Agent 通过 `Client.get_completion()` 或 `Client.get_completion_with_tools()` 调用 LLM，两者内部均走 `litellm.completion()`。
 2. **`drop_params=True`** — LiteLLM 自动丢弃模型不支持的参数（如 `reasoning_effort`、`parallel_tool_calls`），不会报错。
-3. **图片降级** — 如果模型不支持图片，`_is_image_deserialize_error()` 捕获异常后自动剥离图片重试。
+3. **图片降级** — 如果模型不支持图片，`_is_image_deserialize_error()` 捕获异常后自动剥离图片重试；对于信息不明确的上游请求失败，仅在原请求确实包含图片时降级。
 4. **无前端预检查** — 前端始终允许用户上传图片；后端自动处理。
 
 ## Client API

package.json

@@ -9,7 +9,9 @@
         "dompurify": "^3.4.2",
         "markdown-it": "^14.3.0",
         "linkify-it": "^5.0.2",
-        "undici": "^7.28.0"
+        "undici": "^7.28.0",
+        "immutable": "^5.1.9",
+        "uuid": "^11.1.1"
     },
     "dependencies": {
         "@azure/msal-browser": "^5.6.3",

py-src/data_formulator/agents/client_utils.py

@@ -295,10 +295,32 @@ def _strip_images_from_messages(self, messages):
                 sanitized_messages.append(msg)
         return sanitized_messages
 
-    def _is_image_deserialize_error(self, error_text: str) -> bool:
+    def _messages_contain_images(self, messages) -> bool:
+        """Return whether messages contain an image_url content block."""
+        return any(
+            isinstance(msg, dict)
+            and isinstance(msg.get("content"), list)
+            and any(
+                isinstance(item, dict) and item.get("type") == "image_url"
+                for item in msg["content"]
+            )
+            for msg in messages
+        )
+
+    def _is_image_deserialize_error(self, error_text: str, has_images: bool = False) -> bool:
         """Detect provider errors caused by image blocks on text-only models."""
         lowered = error_text.lower()
-        return ("image_url" in lowered and "expected `text`" in lowered) or "unknown variant `image_url`" in lowered
+        return (
+            ("image_url" in lowered and "expected `text`" in lowered)
+            or "unknown variant `image_url`" in lowered
+            or (
+                has_images
+                and (
+                    "upstream request failed" in lowered
+                    or ("image" in lowered and ("not support" in lowered or "unsupported" in lowered))
+                )
+            )
+        )
 
     def _is_reasoning_effort_error(self, error_text: str) -> bool:
         """Detect provider errors caused by an unsupported ``reasoning_effort``
@@ -393,7 +415,7 @@ def get_completion(self, messages, stream=False, reasoning_effort="low",
             if self._is_reasoning_effort_error(err):
                 params.pop("reasoning_effort", None)
                 return self._dispatch(messages=messages, stream=stream, params=params)
-            if self._is_image_deserialize_error(err):
+            if self._is_image_deserialize_error(err, self._messages_contain_images(messages)):
                 sanitized = self._strip_images_from_messages(messages)
                 return self._dispatch(messages=sanitized, stream=stream, params=params)
             raise
@@ -416,7 +438,7 @@ def get_completion_with_tools(self, messages, tools, stream=False,
                 params.pop("reasoning_effort", None)
                 return self._dispatch(messages=messages, stream=stream,
                                       params=params, tools=tools, extra=kwargs)
-            if self._is_image_deserialize_error(err):
+            if self._is_image_deserialize_error(err, self._messages_contain_images(messages)):
                 sanitized = self._strip_images_from_messages(messages)
                 return self._dispatch(messages=sanitized, stream=stream,
                                       params=params, tools=tools, extra=kwargs)

py-src/data_formulator/data_loader/__init__.py

@@ -57,7 +57,7 @@
 # ---------------------------------------------------------------------------
 _LOADER_SPECS: list[tuple[str, str, str, str]] = [
     ("mysql",      "data_formulator.data_loader.mysql_data_loader",      "MySQLDataLoader",      "pymysql"),
-    ("mssql",      "data_formulator.data_loader.mssql_data_loader",      "MSSQLDataLoader",      "pyodbc"),
+    ("mssql",      "data_formulator.data_loader.mssql_data_loader",      "MSSQLDataLoader",      "mssql-python"),
     ("postgresql", "data_formulator.data_loader.postgresql_data_loader",  "PostgreSQLDataLoader", "psycopg2-binary"),
     ("kusto",      "data_formulator.data_loader.kusto_data_loader",      "KustoDataLoader",      "azure-kusto-data"),
     ("databricks", "data_formulator.data_loader.databricks_data_loader", "DatabricksDataLoader", "databricks-sql-connector"),

py-src/data_formulator/data_loader/connector_errors.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+import re
 from typing import Any
 
 from data_formulator.data_loader.external_data_loader import ConnectorParamError
@@ -42,6 +43,21 @@ def classify_connector_error(error: Exception, *, operation: str = "") -> Connec
     text = raw_detail.lower()
     http_status = _http_status(chain)
 
+    firewall_match = re.search(
+        r"client with ip address ['\"]?([0-9a-f:.]+)['\"]? is not allowed to access the server",
+        raw_detail,
+        flags=re.IGNORECASE,
+    )
+    if firewall_match:
+        client_ip = firewall_match.group(1)
+        return ConnectorErrorInfo(
+            ErrorCode.DB_FIREWALL_BLOCKED,
+            f"Azure SQL blocked this network address ({client_ip}). Ask the "
+            "database administrator to add this IP to the SQL server firewall, "
+            "then try again.",
+            detail=safe_detail,
+        )
+
     if any(isinstance(exc, ConnectorParamError) for exc in chain):
         return ConnectorErrorInfo(
             ErrorCode.INVALID_REQUEST,

py-src/data_formulator/data_loader/mssql_data_loader.py

@@ -1,36 +1,17 @@
 import json
 import logging
 import math
-import struct
 from typing import Any
 
+import mssql_python
 import pyarrow as pa
-import pyodbc
 
 from data_formulator.data_loader.external_data_loader import ExternalDataLoader, CatalogNode, MAX_IMPORT_ROWS, sanitize_table_name
 from data_formulator.data_loader import probe_utils
 from data_formulator.datalake.parquet_utils import df_to_safe_records
 
 log = logging.getLogger(__name__)
 
-# ODBC connection attribute for passing a pre-fetched Entra ID (Azure AD)
-# access token to the SQL Server driver (SQL_COPT_SS_ACCESS_TOKEN).
-_SQL_COPT_SS_ACCESS_TOKEN = 1256
-
-# Token audience/scope for Azure SQL / SQL Server Entra ID authentication.
-_AZURE_SQL_SCOPE = "https://database.windows.net/.default"
-
-
-def _is_nan(value) -> bool:
-    """Check if a value is NaN (works for float, int, None)."""
-    if value is None:
-        return True
-    try:
-        return math.isnan(float(value))
-    except (TypeError, ValueError):
-        return False
-
-
 class MSSQLDataLoader(ExternalDataLoader):
     DISPLAY_NAME = "SQL Server"
     DESCRIPTION = "Connect to a Microsoft SQL Server database to query tables with SQL."
@@ -79,20 +60,13 @@ def list_params() -> list[dict[str, Any]]:
                 "tier": "connection",
                 "description": "SQL Server port (default: 1433)",
             },
-            {
-                "name": "driver",
-                "type": "string",
-                "required": False,
-                "default": "ODBC Driver 17 for SQL Server",
-                "tier": "connection",
-                "description": "ODBC driver name",
-            },
             {
                 "name": "encrypt",
                 "type": "string",
                 "required": False,
                 "default": "yes",
                 "tier": "connection",
+                "advanced": True,
                 "description": "Enable encryption (yes/no)",
             },
             {
@@ -101,6 +75,7 @@ def list_params() -> list[dict[str, Any]]:
                 "required": False,
                 "default": "no",
                 "tier": "connection",
+                "advanced": True,
                 "description": "Trust server certificate (yes/no)",
             },
             {
@@ -109,6 +84,7 @@ def list_params() -> list[dict[str, Any]]:
                 "required": False,
                 "default": "30",
                 "tier": "connection",
+                "advanced": True,
                 "description": "Connection timeout in seconds",
             },
         ]
@@ -180,8 +156,7 @@ def auth_instructions() -> str:
 
 **Windows authentication (Windows only):** Choose *Windows authentication* and leave username/password empty.
 
-**Prerequisites (macOS/Linux only):**
-Install the ODBC driver: `brew install unixodbc msodbcsql18` (macOS) or `sudo apt-get install unixodbc-dev msodbcsql18` (Ubuntu/Debian). For Entra ID you also need the Azure CLI (`brew install azure-cli`) and to run `az login`.
+**Microsoft Entra prerequisite:** Install the Azure CLI and run `az login`. The SQL Server driver is included with Data Formulator; no separate ODBC driver installation is needed.
 
 **Troubleshooting:** Confirm you are signed in with `az account show`. Ensure the SQL Server service is running and TCP/IP is enabled. Test SQL auth with `sqlcmd -S <server> -d <database> -U <user> -P <password>`."""
 
@@ -196,7 +171,6 @@ def __init__(self, params: dict[str, Any]):
         self.user = params.get("user", "").strip()
         self.password = params.get("password", "").strip()
         self.port = params.get("port", "1433")
-        self.driver = params.get("driver", "ODBC Driver 17 for SQL Server")
         self.encrypt = params.get("encrypt", "yes")
         self.trust_server_certificate = params.get("trust_server_certificate", "no")
         self.connection_timeout = params.get("connection_timeout", "30")
@@ -206,66 +180,34 @@ def __init__(self, params: dict[str, Any]):
         # When no database specified, connect to master for catalog browsing
         connect_db = self.database or "master"
 
-        # Build the auth-independent part of the ODBC connection string.
+        # Build the auth-independent connection string. mssql-python uses
+        # Direct Database Connectivity, so no external ODBC driver is needed.
         conn_str = (
-            f"DRIVER={{{self.driver}}};"
             f"SERVER={self.server},{self.port};"
             f"DATABASE={connect_db};"
             f"Encrypt={self.encrypt};"
             f"TrustServerCertificate={self.trust_server_certificate};"
-            f"Connection Timeout={self.connection_timeout};"
         )
 
-        connect_kwargs: dict[str, Any] = {}
+        try:
+            connection_timeout = max(1, int(self.connection_timeout))
+        except (TypeError, ValueError) as e:
+            raise ValueError("Connection timeout must be a positive number of seconds") from e
+
         if self.auth_path == "entra_id":
-            # Entra ID: fetch a token via DefaultAzureCredential (az login /
-            # Managed Identity / VS Code / env) and hand it to the driver.
-            # No UID/PWD/Trusted_Connection goes into the string.
-            connect_kwargs["attrs_before"] = {
-                _SQL_COPT_SS_ACCESS_TOKEN: self._get_entra_id_token_struct()
-            }
+            conn_str += "Authentication=ActiveDirectoryDefault;"
         elif self.user or self.auth_path == "sql_auth":
-            conn_str += f"UID={self.user};PWD={self.password};"
+            conn_str += f"Authentication=SqlPassword;UID={self.user};PWD={self.password};"
         else:
             conn_str += "Trusted_Connection=yes;"
 
         try:
-            self._conn = pyodbc.connect(conn_str, **connect_kwargs)
+            self._conn = mssql_python.connect(conn_str, timeout=connection_timeout)
             log.info(f"Successfully connected to SQL Server: {self.server}/{self.database}")
         except Exception as e:
             log.error(f"Failed to connect to SQL Server: {e}")
             raise ValueError(f"Failed to connect to SQL Server '{self.server}': {e}") from e
 
-    @staticmethod
-    def _get_entra_id_token_struct() -> bytes:
-        """Fetch an Entra ID access token for Azure SQL and pack it for ODBC.
-
-        Uses ``DefaultAzureCredential`` so a local ``az login`` (or Managed
-        Identity / VS Code / environment credentials) is enough to connect.
-        The token is packed into the little-endian, length-prefixed UTF-16-LE
-        struct expected by the SQL Server ODBC access-token attribute.
-        """
-        try:
-            from azure.identity import DefaultAzureCredential
-        except ImportError as e:
-            raise ValueError(
-                "Microsoft Entra ID authentication requires the 'azure-identity' "
-                "package. Install it with `uv pip install azure-identity`."
-            ) from e
-
-        try:
-            credential = DefaultAzureCredential()
-            token = credential.get_token(_AZURE_SQL_SCOPE).token
-        except Exception as e:
-            raise ValueError(
-                "Failed to acquire a Microsoft Entra ID token. Run `az login` in "
-                "your terminal (or configure a Managed Identity), then retry. "
-                f"Details: {e}"
-            ) from e
-
-        token_bytes = token.encode("UTF-16-LE")
-        return struct.pack(f"<I{len(token_bytes)}s", len(token_bytes), token_bytes)
-
     # SQL Server types that may need special handling
     _CX_SPATIAL_TYPES = {'geometry', 'geography'}  # use .STAsText()
     _CX_OTHER_UNSUPPORTED = {'hierarchyid', 'xml', 'sql_variant', 'image', 'timestamp'}

py-src/data_formulator/datalake/parquet_utils.py

@@ -101,11 +101,36 @@ def df_to_safe_records(df: pd.DataFrame) -> list[dict[str, Any]]:
     streaming should call this function rather than using ``df.to_json``
     / ``df.to_dict`` directly.
     """
+    safe_df = df.copy(deep=False)
+    for column in safe_df.select_dtypes(include=["object", "string"]).columns:
+        safe_df[column] = safe_df[column].map(_repair_invalid_unicode)
     return json.loads(
-        df.to_json(orient="records", date_format="iso", default_handler=str)
+        safe_df.to_json(orient="records", date_format="iso", default_handler=str)
     )
 
 
+def _repair_invalid_unicode(value: Any) -> Any:
+    """Replace malformed surrogate code points while preserving valid Unicode."""
+    if isinstance(value, str):
+        try:
+            value.encode("utf-8")
+            return value
+        except UnicodeEncodeError:
+            return value.encode("utf-8", errors="replace").decode("utf-8")
+    if isinstance(value, dict):
+        return {
+            _repair_invalid_unicode(key): _repair_invalid_unicode(item)
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [_repair_invalid_unicode(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(_repair_invalid_unicode(item) for item in value)
+    if isinstance(value, set):
+        return {_repair_invalid_unicode(item) for item in value}
+    return value
+
+
 def normalize_dtype_to_app_type(dtype_str: str) -> str:
     """Map a pandas/Arrow dtype string to a standardized App Type label.
 

py-src/data_formulator/errors.py

@@ -50,6 +50,7 @@ class ErrorCode:
 
     # --- Data / connector ---
     CONNECTOR_AUTH_FAILED = "CONNECTOR_AUTH_FAILED"
+    DB_FIREWALL_BLOCKED = "DB_FIREWALL_BLOCKED"
     DB_CONNECTION_FAILED = "DB_CONNECTION_FAILED"
     DB_QUERY_ERROR = "DB_QUERY_ERROR"
     DATA_LOAD_ERROR = "DATA_LOAD_ERROR"