diff --git a/.env.template b/.env.template index eb407501..0d779c9c 100644 --- a/.env.template +++ b/.env.template @@ -29,6 +29,14 @@ SANDBOX=local # code execution backend: 'local' (default) or 'docke # .env to preserve sessions. See DEVELOPMENT.md "Server Migration Checklist". # FLASK_SECRET_KEY= +# Kusto delegated Microsoft sign-in (Authorization Code + PKCE). +# Register a public-client Entra application for your deployment. Client and +# tenant IDs are public identifiers, but they are deployment-specific and +# should not be copied from another organization. No client secret is needed. +# KUSTO_OAUTH_CLIENT_ID= +# KUSTO_OAUTH_TENANT_ID= +# KUSTO_OAUTH_REDIRECT_URI=http://localhost:5567/api/auth/kusto/callback + # Data directory — where workspaces and user data are stored on disk. # Useful for server deployments with large datasets or dedicated storage volumes. # Resolution order: --data-dir CLI flag > DATA_FORMULATOR_HOME env var > ~/.data_formulator diff --git a/README.md b/README.md index 28d40020..576816a2 100644 --- a/README.md +++ b/README.md @@ -43,14 +43,17 @@ https://github.com/user-attachments/assets/8e4f8a08-6423-4227-a1f7-559e0126ce31 ## News 🔥🔥🔥 -[07-17-2026] **Data Formulator 0.8 alpha 2** — A more connected, conversational way to work with data: +[07-23-2026] **Data Formulator 0.8 alpha** (a1–a3, latest: 0.8.0a3) includes: -- **Explore large datasets through conversation.** Connect a database, then ask the agent to find the right tables, filter the data, or update your selection as your question evolves. You can review the resulting filters, previews, and data sources before anything is loaded. -- **Keep the whole analysis in one conversation.** Agents can load data without losing track of what you asked. Questions, explanations, and results stay together in the Data Thread, so you can pick up from any step or branch into a new question, column, or chart. -- **Choose a chart that fits the question.** The gallery now includes bullet, connected scatter, ECDF, Gantt, range area, slope, sparkline, and violin charts, along with better recommendations. Files you attach also stay available to the analyst as the exploration continues. -- **Spend less time troubleshooting.** This release improves long-running sessions, model routing, data isolation, installation across platforms, dependency security, and MySQL data freshness. Persistent logs and an in-app log viewer make problems easier to track down. +- **Conversational database loading.** Agents can discover relevant tables, propose filters, preview results, and revise a loading plan through conversation before importing data. +- **Unified Data Thread.** Questions, clarifications, explanations, tables, and charts share one conversation history, with branching from earlier steps into new questions, calculated columns, or visualizations. +- **Expanded chart gallery, powered by Flint.** New bullet, connected scatter, ECDF, Gantt, range area, slope, sparkline, and violin charts, along with improved chart recommendations. Try the open-source [Flint chart language](https://microsoft.github.io/flint-chart/) in your own applications. +- **Persistent analyst attachments.** CSV, JSON, Excel, and other attached files remain available to the analyst throughout an exploration instead of being embedded once in a prompt. +- **Databricks connector.** Browse Unity Catalog catalogs, schemas, and tables, then load Databricks data into the exploration workflow. +- **Microsoft authentication for enterprise connectors.** SQL Server supports passwordless Microsoft Entra ID authentication through `az login`, including an in-app flow for local deployments. Kusto supports delegated Microsoft sign-in alongside Azure default identity and service principal authentication. +- **Connector setup and diagnostics.** Connection forms separate connection, scope, and source-specific authentication options. Persistent server logs and an in-app log viewer help diagnose failures. -> Preview with `pip install --pre data_formulator==0.8.0a2` or `uvx --from data_formulator==0.8.0a2 data_formulator`. +> Preview with `pip install --pre data_formulator==0.8.0a3` or `uvx data_formulator@0.8.0a3`. > Install the latest stable release (0.7) with `pip install data_formulator` or run instantly with `uvx data_formulator`. diff --git a/py-src/data_formulator/app.py b/py-src/data_formulator/app.py index dde1a5c0..02e1a10d 100644 --- a/py-src/data_formulator/app.py +++ b/py-src/data_formulator/app.py @@ -295,6 +295,10 @@ def _register_blueprints(): from data_formulator.auth.gateways.oidc_gateway import auth_tokens_bp app.register_blueprint(auth_tokens_bp) + # Kusto delegated Microsoft sign-in is independent of application SSO. + from data_formulator.auth.gateways.kusto_oauth_gateway import kusto_oauth_bp + app.register_blueprint(kusto_oauth_bp) + # Register backend OIDC gateway (auto-detected: active when OIDC_CLIENT_SECRET is set) from data_formulator.auth.providers.oidc import is_backend_oidc_mode if is_backend_oidc_mode(): diff --git a/py-src/data_formulator/auth/gateways/kusto_oauth_gateway.py b/py-src/data_formulator/auth/gateways/kusto_oauth_gateway.py new file mode 100644 index 00000000..f09eda8d --- /dev/null +++ b/py-src/data_formulator/auth/gateways/kusto_oauth_gateway.py @@ -0,0 +1,252 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Microsoft delegated OAuth flow for Kusto connector access.""" + +from __future__ import annotations + +import base64 +import hashlib +import html +import json +import os +import re +import secrets +import time +import urllib.parse +from typing import Any + +import requests as http +from flask import Blueprint, Response, request, session + +from data_formulator.errors import AppError, ErrorCode + + +kusto_oauth_bp = Blueprint( + "kusto_oauth", __name__, url_prefix="/api/auth/kusto", +) + +_STATE_KEY = "kusto_oauth_state" +_KUSTO_HOST_SUFFIXES = ( + ".kusto.windows.net", + ".kusto.chinacloudapi.cn", + ".kusto.usgovcloudapi.net", +) +_LOGIN_HOSTS = { + "login.microsoftonline.com", + "login.microsoftonline.us", + "login.chinacloudapi.cn", + "login.windows.net", +} + + +def _oauth_config(authority_host: str | None = None) -> dict[str, str]: + tenant_id = os.environ.get("KUSTO_OAUTH_TENANT_ID", "organizations") + if not re.fullmatch(r"[A-Za-z0-9.-]+", tenant_id): + raise AppError(ErrorCode.INVALID_REQUEST, "Invalid Kusto OAuth tenant") + authority_host = os.environ.get("KUSTO_OAUTH_AUTHORITY_HOST") or authority_host + authority_host = (authority_host or "https://login.microsoftonline.com").rstrip("/") + parsed_authority = urllib.parse.urlparse(authority_host) + if parsed_authority.scheme != "https" or parsed_authority.hostname not in _LOGIN_HOSTS: + raise AppError(ErrorCode.INVALID_REQUEST, "Invalid Kusto OAuth authority") + return { + "client_id": os.environ.get("KUSTO_OAUTH_CLIENT_ID", ""), + "client_secret": os.environ.get("KUSTO_OAUTH_CLIENT_SECRET", ""), + "authorize_url": f"{authority_host}/{tenant_id}/oauth2/v2.0/authorize", + "token_url": f"{authority_host}/{tenant_id}/oauth2/v2.0/token", + } + + +def _callback_url() -> str: + return os.environ.get( + "KUSTO_OAUTH_REDIRECT_URI", + request.url_root.rstrip("/") + "/api/auth/kusto/callback", + ) + + +def _normalize_cluster(cluster: str) -> str: + parsed = urllib.parse.urlparse(cluster.strip()) + hostname = (parsed.hostname or "").lower() + if ( + parsed.scheme != "https" + or not hostname.endswith(_KUSTO_HOST_SUFFIXES) + or parsed.username + or parsed.password + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + ): + raise AppError( + ErrorCode.INVALID_REQUEST, + "Enter a valid HTTPS Azure Data Explorer cluster URL", + ) + origin = f"https://{hostname}" + if parsed.port: + origin += f":{parsed.port}" + return origin + + +def _cluster_auth_metadata(cluster: str) -> tuple[str, str]: + """Return the SDK-compatible resource scope and login endpoint.""" + origin = _normalize_cluster(cluster) + try: + response = http.get( + f"{origin}/v1/rest/auth/metadata", + allow_redirects=False, + timeout=10, + ) + response.raise_for_status() + azure_ad = response.json()["AzureAD"] + resource = str(azure_ad["KustoServiceResourceId"]).rstrip("/") + login_endpoint = str(azure_ad["LoginEndpoint"]).rstrip("/") + resource_url = urllib.parse.urlparse(resource) + login_url = urllib.parse.urlparse(login_endpoint) + if ( + resource_url.scheme != "https" + or not (resource_url.hostname or "").endswith(_KUSTO_HOST_SUFFIXES) + or login_url.scheme != "https" + or login_url.hostname not in _LOGIN_HOSTS + ): + raise ValueError("untrusted Kusto authentication metadata") + return f"{resource}/.default", login_endpoint + except Exception: + if origin.endswith(".kusto.windows.net"): + return "https://kusto.kusto.windows.net/.default", "https://login.microsoftonline.com" + raise AppError( + ErrorCode.SERVICE_UNAVAILABLE, + "Could not discover authentication settings for this Kusto cloud", + ) + + +def _frontend_origin(value: str) -> str: + parsed = urllib.parse.urlparse(value) + origin = f"{parsed.scheme}://{parsed.netloc}" + request_origin = request.url_root.rstrip("/") + configured = { + item.strip().rstrip("/") + for item in os.environ.get("KUSTO_OAUTH_ALLOWED_ORIGINS", "").split(",") + if item.strip() + } + is_local = parsed.scheme == "http" and parsed.hostname in {"localhost", "127.0.0.1"} + if not parsed.netloc or parsed.path or parsed.params or parsed.query or parsed.fragment: + raise AppError(ErrorCode.INVALID_REQUEST, "Invalid Data Formulator origin") + if origin != request_origin and origin not in configured and not is_local: + raise AppError(ErrorCode.INVALID_REQUEST, "Data Formulator origin is not allowed") + return origin + + +def _pkce_challenge(verifier: str) -> str: + digest = hashlib.sha256(verifier.encode("ascii")).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def _popup_response(origin: str, payload: dict[str, Any]) -> Response: + nonce = secrets.token_urlsafe(16) + payload_json = json.dumps(payload).replace("<", "\\u003c") + origin_json = json.dumps(origin).replace("<", "\\u003c") + body = f""" +Microsoft sign-in +

You can close this window.

""" + response = Response(body, content_type="text/html; charset=utf-8") + response.headers["Cache-Control"] = "no-store" + response.headers["Content-Security-Policy"] = f"default-src 'none'; script-src 'nonce-{nonce}'" + return response + + +@kusto_oauth_bp.route("/login") +def kusto_login(): + """Start an Authorization Code + PKCE flow for the selected cluster.""" + scope, login_endpoint = _cluster_auth_metadata( + request.args.get("kusto_cluster", ""), + ) + config = _oauth_config(login_endpoint) + if not config["client_id"]: + raise AppError(ErrorCode.SERVICE_UNAVAILABLE, "Kusto Microsoft sign-in is not configured") + + origin = _frontend_origin(request.args.get("df_origin", "")) + state = secrets.token_urlsafe(32) + verifier = secrets.token_urlsafe(64) + states = { + key: value + for key, value in session.get(_STATE_KEY, {}).items() + if time.time() - value.get("created_at", 0) <= 600 + } + states[state] = { + "verifier": verifier, + "origin": origin, + "scope": scope, + "token_url": config["token_url"], + "created_at": time.time(), + } + session[_STATE_KEY] = states + + params = { + "client_id": config["client_id"], + "response_type": "code", + "redirect_uri": _callback_url(), + "response_mode": "query", + "scope": f"openid profile offline_access {scope}", + "state": state, + "code_challenge": _pkce_challenge(verifier), + "code_challenge_method": "S256", + "prompt": "select_account", + } + location = f"{config['authorize_url']}?{urllib.parse.urlencode(params)}" + response = Response(status=302) + response.headers["Location"] = location + return response + + +@kusto_oauth_bp.route("/callback") +def kusto_callback(): + """Exchange the Microsoft authorization code and notify the opener.""" + state = request.args.get("state", "") + states = session.get(_STATE_KEY, {}) + pending = states.pop(state, None) + session[_STATE_KEY] = states + if not pending or time.time() - pending.get("created_at", 0) > 600: + raise AppError(ErrorCode.INVALID_REQUEST, "Invalid or expired Kusto OAuth state") + + origin = pending["origin"] + idp_error = request.args.get("error") + if idp_error: + return _popup_response(origin, { + "type": "df-sso-auth", + "error": "Microsoft sign-in failed", + }) + + code = request.args.get("code") + if not code: + return _popup_response(origin, { + "type": "df-sso-auth", + "error": "Microsoft did not return an authorization code", + }) + + config = _oauth_config() + token_data = { + "client_id": config["client_id"], + "grant_type": "authorization_code", + "code": code, + "redirect_uri": _callback_url(), + "scope": f"openid profile offline_access {pending['scope']}", + "code_verifier": pending["verifier"], + } + if config["client_secret"]: + token_data["client_secret"] = config["client_secret"] + token_response = http.post(pending["token_url"], data=token_data, timeout=15) + if not token_response.ok: + return _popup_response(origin, { + "type": "df-sso-auth", + "error": "Microsoft token exchange failed", + }) + + tokens = token_response.json() + return _popup_response(origin, { + "type": "df-sso-auth", + "access_token": tokens["access_token"], + "refresh_token": tokens.get("refresh_token"), + "expires_in": tokens.get("expires_in", 3600), + }) \ No newline at end of file diff --git a/py-src/data_formulator/data_connector.py b/py-src/data_formulator/data_connector.py index 933fa805..0ef27bf5 100644 --- a/py-src/data_formulator/data_connector.py +++ b/py-src/data_formulator/data_connector.py @@ -480,10 +480,14 @@ def _resolve_delegated_login(self) -> dict[str, Any] | None: return None login_url = raw.get("login_url", "") # Resolve relative URLs to the connector's API prefix - if login_url and not login_url.startswith("http"): + if login_url and not login_url.startswith(("http", "/")): login_url = f"/api/connectors/{self._source_id}/{login_url}" # Only send safe fields to the frontend - return {"login_url": login_url, "label": raw.get("label", "")} + return { + "login_url": login_url, + "label": raw.get("label", ""), + "params": list(raw.get("params", [])), + } # -- Identity + Loader Management -------------------------------------- @@ -1070,6 +1074,117 @@ def pick_local_directory(): return json_ok({"path": folder}) +def _az_account_summary() -> dict[str, Any] | None: + """Return the currently signed-in Azure CLI account, or None. + + Runs ``az account show`` (no shell) and parses the JSON output. Returns + ``None`` when the CLI is missing, the user is not signed in, or the call + fails/times out. + """ + import json as _json + import shutil + import subprocess + + if shutil.which("az") is None: + return None + try: + result = subprocess.run( + ["az", "account", "show", "--only-show-errors", "-o", "json"], + capture_output=True, text=True, timeout=15, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if result.returncode != 0 or not result.stdout.strip(): + return None + try: + data = _json.loads(result.stdout) + except ValueError: + return None + user = data.get("user") or {} + return { + "user": user.get("name"), + "subscription": data.get("name"), + "tenant_id": data.get("tenantId"), + } + + +@connectors_bp.route("/api/local/azure-status", methods=["POST"]) +def azure_cli_status(): + """Report whether the local Azure CLI is installed and signed in. + + Only available in local deployment mode. Used by connectors that support + Microsoft Entra ID auth (e.g. SQL Server) to show the current sign-in + state next to an in-app "Sign in with Azure CLI" button. + """ + import shutil + + from data_formulator.auth.identity import is_local_mode + + if not is_local_mode(): + raise AppError(ErrorCode.ACCESS_DENIED, "Not available in server mode") + + if shutil.which("az") is None: + return json_ok({"installed": False, "signed_in": False, "account": None}) + + account = _az_account_summary() + return json_ok({ + "installed": True, + "signed_in": account is not None, + "account": account, + }) + + +@connectors_bp.route("/api/local/azure-login", methods=["POST"]) +def azure_cli_login(): + """Run ``az login`` on the local machine, opening the system browser. + + Only available in local deployment mode (the backend runs on the user's + own machine, so the browser opens for them). Blocks until the interactive + sign-in completes or times out. On success returns the signed-in account. + """ + import shutil + import subprocess + + from data_formulator.auth.identity import is_local_mode + + if not is_local_mode(): + raise AppError(ErrorCode.ACCESS_DENIED, "Not available in server mode") + + if shutil.which("az") is None: + raise AppError( + ErrorCode.CONNECTOR_ERROR, + "Azure CLI ('az') is not installed. Install it (e.g. " + "`brew install azure-cli`) or run `az login` in a terminal.", + ) + + # Already signed in — treat as success without opening a browser. + existing = _az_account_summary() + if existing is not None: + return json_ok({"signed_in": True, "account": existing}) + + try: + result = subprocess.run( + ["az", "login", "--only-show-errors", "-o", "json"], + capture_output=True, text=True, timeout=300, + ) + except subprocess.TimeoutExpired: + raise AppError( + ErrorCode.CONNECTOR_ERROR, + "Azure sign-in timed out. Complete the browser prompt and retry.", + ) + except OSError as exc: + logger.warning("Failed to launch az login: %s", exc) + raise AppError(ErrorCode.CONNECTOR_ERROR, "Failed to launch Azure CLI sign-in") + + if result.returncode != 0: + detail = (result.stderr or "").strip().splitlines() + message = detail[-1] if detail else "Azure sign-in failed or was cancelled." + raise AppError(ErrorCode.CONNECTOR_ERROR, message) + + account = _az_account_summary() + return json_ok({"signed_in": account is not None, "account": account}) + + @connectors_bp.route("/api/connectors", methods=["GET"]) def list_connectors(): """List all registered connector instances (admin + user) with connection status.""" @@ -1424,6 +1539,7 @@ def connector_connect(): **extra_params, "access_token": access_token, "refresh_token": data.get("refresh_token", ""), + "token_expires_at": time.time() + float(data.get("expires_in") or 3600), } else: user_params = data.get("params", {}) diff --git a/py-src/data_formulator/data_loader/__init__.py b/py-src/data_formulator/data_loader/__init__.py index 56a6e3b3..c024f411 100644 --- a/py-src/data_formulator/data_loader/__init__.py +++ b/py-src/data_formulator/data_loader/__init__.py @@ -60,6 +60,7 @@ ("mssql", "data_formulator.data_loader.mssql_data_loader", "MSSQLDataLoader", "pyodbc"), ("postgresql", "data_formulator.data_loader.postgresql_data_loader", "PostgreSQLDataLoader", "psycopg2-binary"), ("kusto", "data_formulator.data_loader.kusto_data_loader", "KustoDataLoader", "azure-kusto-data"), + ("databricks", "data_formulator.data_loader.databricks_data_loader", "DatabricksDataLoader", "databricks-sql-connector"), ("s3", "data_formulator.data_loader.s3_data_loader", "S3DataLoader", "boto3"), ("azure_blob", "data_formulator.data_loader.azure_blob_data_loader", "AzureBlobDataLoader", "azure-storage-blob"), ("mongodb", "data_formulator.data_loader.mongodb_data_loader", "MongoDBDataLoader", "pymongo"), diff --git a/py-src/data_formulator/data_loader/databricks_data_loader.py b/py-src/data_formulator/data_loader/databricks_data_loader.py new file mode 100644 index 00000000..41034cb0 --- /dev/null +++ b/py-src/data_formulator/data_loader/databricks_data_loader.py @@ -0,0 +1,354 @@ +import logging +import os +from typing import Any + +import pyarrow as pa + +from data_formulator.data_loader.external_data_loader import ( + ExternalDataLoader, + MAX_IMPORT_ROWS, + build_where_clause_inline, + _esc_id, + _esc_str, +) + +from databricks import sql as databricks_sql + +logger = logging.getLogger(__name__) + +# Unity Catalog exposes an ``information_schema`` schema inside every catalog; +# it is metadata, not user data, so it is hidden from browsing. +_HIDDEN_SCHEMAS = {"information_schema"} +# ``system`` holds platform telemetry and ``__databricks_internal`` is internal. +_HIDDEN_CATALOGS = {"system", "__databricks_internal"} + +# Bound broad (unpinned) catalog scans so browsing a large metastore stays fast. +_MAX_CATALOGS = 10 +_MAX_TABLES = 500 + + +def _bt(name: str) -> str: + """Backtick-quote a Databricks/Spark SQL identifier.""" + return _esc_id(name, "`") + + +class DatabricksDataLoader(ExternalDataLoader): + """Databricks SQL loader for browsing and importing Unity Catalog data. + + Connects to a Databricks SQL warehouse via ``databricks-sql-connector`` and + browses the Unity Catalog three-level namespace (``catalog.schema.table``). + Data is fetched Arrow-native via the connector's ``fetchall_arrow()``. + """ + + DISPLAY_NAME = "Databricks" + DESCRIPTION = "Query Databricks Unity Catalog tables through a SQL warehouse." + + @staticmethod + def list_params() -> list[dict[str, Any]]: + return [ + {"name": "server_hostname", "type": "string", "required": True, "tier": "connection", + "description": "e.g., adb-1234567890.11.azuredatabricks.net"}, + {"name": "http_path", "type": "string", "required": True, "tier": "connection", + "description": "SQL warehouse HTTP path, e.g., /sql/1.0/warehouses/abc123"}, + {"name": "catalog", "type": "string", "required": False, "tier": "connection", + "description": "Unity Catalog name (leave empty to browse all catalogs)"}, + {"name": "schema", "type": "string", "required": False, "tier": "filter", + "description": "Schema name (leave empty to browse all schemas in the catalog)"}, + {"name": "access_token", "type": "string", "required": False, "sensitive": True, "tier": "auth", + "description": "Databricks personal access token (dapi...)"}, + ] + + @classmethod + def auth_paths(cls) -> list[dict[str, Any]]: + # Personal Access Token is the always-available default. A delegated + # "Sign in with Databricks" (OAuth U2M) path is surfaced only when the + # server is configured for it; the gateway is a TODO (see + # delegated_login_config), so this stays dormant until wired up. + oauth_ready = bool(os.environ.get("DATABRICKS_OAUTH_CLIENT_ID")) + paths = [ + { + "id": "token", + "label": "Personal access token", + "description": "Use a Databricks personal access token (dapi…).", + "fields": ["access_token"], + "required_fields": ["access_token"], + "kind": "credentials", + "default": not oauth_ready, + }, + ] + if oauth_ready: + paths.insert(0, { + "id": "databricks_sign_in", + "label": "Sign in with Databricks", + "description": "Use your Databricks identity and existing Unity Catalog permissions.", + "fields": [], + "required_fields": [], + "kind": "delegated_login", + "default": True, + }) + return paths + + @classmethod + def infer_auth_path(cls, params: dict[str, Any]) -> str: + if params.get("access_token"): + return "token" + if os.environ.get("DATABRICKS_OAUTH_CLIENT_ID"): + return "databricks_sign_in" + return "token" + + @staticmethod + def delegated_login_config() -> dict[str, Any] | None: + # TODO: implement databricks_oauth_gateway.py (Authorization Code + PKCE + # against the workspace OAuth endpoint), then return its login URL here. + return None + + @staticmethod + def auth_instructions() -> str: + return """**Example:** server_hostname: `adb-1234567890.11.azuredatabricks.net` · http_path: `/sql/1.0/warehouses/abc123` + +**Connection:** Find both values on your SQL warehouse's **Connection details** tab in the Databricks workspace (SQL → SQL Warehouses → your warehouse). + +**Personal access token:** In Databricks, open **Settings → Developer → Access tokens** and generate a token (starts with `dapi`). The token's user needs `USE CATALOG` / `USE SCHEMA` and `SELECT` on the Unity Catalog objects you want to read. + +**Scope:** Leave *catalog* and *schema* empty to browse everything you can access, or set them to jump straight to a specific catalog/schema.""" + + def __init__(self, params: dict[str, Any]): + self.params = params + self.auth_path = params.get("_auth_path") or self.infer_auth_path(params) + + raw_host = (params.get("server_hostname") or "").strip() + # Accept a full URL or a bare hostname. + raw_host = raw_host.replace("https://", "").replace("http://", "").rstrip("/") + self.server_hostname = raw_host + self.http_path = (params.get("http_path") or "").strip() + self.catalog = (params.get("catalog") or "").strip() + self.schema = (params.get("schema") or "").strip() + self.access_token = params.get("access_token") or "" + + if not self.server_hostname: + raise ValueError("Databricks server_hostname is required") + if not self.http_path: + raise ValueError("Databricks http_path (SQL warehouse) is required") + if not self.access_token: + raise ValueError("Databricks access token is required") + + try: + self._conn = databricks_sql.connect( + server_hostname=self.server_hostname, + http_path=self.http_path, + access_token=self.access_token, + ) + except Exception as e: + logger.error("Failed to connect to Databricks (%s): %s", self.server_hostname, e) + raise ValueError( + f"Failed to connect to Databricks warehouse at '{self.server_hostname}': {e}" + ) from e + + logger.info("Successfully connected to Databricks: %s", self.server_hostname) + + # ------------------------------------------------------------------ # + # Query helpers # + # ------------------------------------------------------------------ # + + def _query_arrow(self, query: str) -> pa.Table: + """Run *query* and return results as a PyArrow Table (Arrow-native).""" + cur = self._conn.cursor() + try: + cur.execute(query) + if cur.description is None: + return pa.table({}) + return cur.fetchall_arrow() + finally: + cur.close() + + def _query_rows(self, query: str) -> list[dict[str, Any]]: + """Run *query* and return a list of row dicts.""" + cur = self._conn.cursor() + try: + cur.execute(query) + if cur.description is None: + return [] + columns = [desc[0] for desc in cur.description] + return [dict(zip(columns, row)) for row in cur.fetchall()] + finally: + cur.close() + + def _resolve_source_table(self, source_table: str) -> tuple[str, str, str]: + """Parse ``catalog.schema.table`` (filling pinned params when partial).""" + parts = source_table.split(".") + if len(parts) >= 3: + return parts[0], parts[1], ".".join(parts[2:]) + if len(parts) == 2: + return self.catalog, parts[0], parts[1] + return self.catalog, self.schema, parts[0] + + # ------------------------------------------------------------------ # + # Catalog hierarchy # + # ------------------------------------------------------------------ # + + @staticmethod + def catalog_hierarchy() -> list[dict[str, str]]: + return [ + {"key": "catalog", "label": "Catalog"}, + {"key": "schema", "label": "Schema"}, + {"key": "table", "label": "Table"}, + ] + + def _catalogs(self) -> list[str]: + if self.catalog: + return [self.catalog] + try: + rows = self._query_rows("SHOW CATALOGS") + except Exception as e: + logger.warning("SHOW CATALOGS failed: %s", e) + return [] + names = [str(r.get("catalog") or r.get("catalog_name") or "") for r in rows] + names = [n for n in names if n and n not in _HIDDEN_CATALOGS] + return names[:_MAX_CATALOGS] + + def list_tables(self, table_filter: str | None = None) -> list[dict[str, Any]]: + """List Unity Catalog tables within the pinned/browsable scope. + + Uses each catalog's ``information_schema`` to batch-fetch tables, + columns and comments (two queries per catalog) so browsing stays fast. + """ + results: list[dict[str, Any]] = [] + for catalog in self._catalogs(): + self._report_progress(f"Listing tables in catalog {catalog}…") + try: + results.extend(self._list_tables_in_catalog(catalog, table_filter)) + except Exception as e: + logger.warning("Skipped catalog '%s': %s", catalog, e) + if len(results) >= _MAX_TABLES: + logger.info("Reached %d table limit, stopping enumeration", _MAX_TABLES) + break + return results[:_MAX_TABLES] + + def _list_tables_in_catalog( + self, catalog: str, table_filter: str | None = None, + ) -> list[dict[str, Any]]: + cat = _bt(catalog) + schema_pred = "" + if self.schema: + schema_pred = f"AND table_schema = '{_esc_str(self.schema)}'" + + tables_rows = self._query_rows(f""" + SELECT table_schema, table_name, comment + FROM {cat}.information_schema.tables + WHERE table_schema NOT IN ('information_schema') + {schema_pred} + ORDER BY table_schema, table_name + """) + + cols_rows = self._query_rows(f""" + SELECT table_schema, table_name, column_name, data_type, comment + FROM {cat}.information_schema.columns + WHERE table_schema NOT IN ('information_schema') + {schema_pred} + ORDER BY table_schema, table_name, ordinal_position + """) + + col_map: dict[str, list[dict[str, Any]]] = {} + for r in cols_rows: + key = f"{r['table_schema']}.{r['table_name']}" + entry: dict[str, Any] = {"name": r["column_name"], "type": r["data_type"]} + comment = r.get("comment") + if comment and str(comment).strip(): + entry["description"] = str(comment).strip() + col_map.setdefault(key, []).append(entry) + + results: list[dict[str, Any]] = [] + for r in tables_rows: + schema = r["table_schema"] + table = r["table_name"] + full_name = f"{catalog}.{schema}.{table}" + if table_filter and table_filter.lower() not in full_name.lower(): + continue + columns = col_map.get(f"{schema}.{table}", []) + metadata: dict[str, Any] = { + "columns": columns, + "source_metadata_status": "synced" if columns else "partial", + } + table_comment = r.get("comment") + if table_comment and str(table_comment).strip(): + metadata["description"] = str(table_comment).strip() + results.append({ + "name": full_name, + "path": [catalog, schema, table], + "metadata": metadata, + }) + return results + + def get_column_types(self, source_table: str) -> dict[str, Any]: + """Return source-level column types/comments for a single table. + + Uses ``DESCRIBE TABLE`` rather than ``information_schema.columns``: the + latter is unreliable for special catalogs (e.g. the built-in + ``samples`` catalog exposes tables but no per-schema column rows), + whereas ``DESCRIBE`` works uniformly across Unity Catalog. + """ + try: + catalog, schema, table = self._resolve_source_table(source_table) + qualified = f"{_bt(catalog)}.{_bt(schema)}.{_bt(table)}" + rows = self._query_rows(f"DESCRIBE TABLE {qualified}") + columns: list[dict[str, Any]] = [] + for r in rows: + name = r.get("col_name") + # DESCRIBE appends partition/detail sections after a blank or + # ``# ...`` separator row — stop at the first such marker. + if not name or str(name).startswith("#"): + break + entry: dict[str, Any] = {"name": name, "type": r.get("data_type")} + comment = r.get("comment") + if comment and str(comment).strip(): + entry["description"] = str(comment).strip() + columns.append(entry) + if columns: + return {"columns": columns} + except Exception as e: + logger.debug("get_column_types failed for %s: %s", source_table, e) + return {} + + # ------------------------------------------------------------------ # + # Data fetch # + # ------------------------------------------------------------------ # + + def fetch_data_as_arrow( + self, + source_table: str, + import_options: dict[str, Any] | None = None, + ) -> pa.Table: + opts = import_options or {} + size = min(opts.get("size", MAX_IMPORT_ROWS), MAX_IMPORT_ROWS) + sort_columns = opts.get("sort_columns") + sort_order = opts.get("sort_order", "asc") + conditions = opts.get("conditions", []) + + if not source_table: + raise ValueError("source_table must be provided") + + catalog, schema, table = self._resolve_source_table(source_table) + qualified = f"{_bt(catalog)}.{_bt(schema)}.{_bt(table)}" + + columns = opts.get("columns") + if columns: + col_list = ", ".join(_bt(c) for c in columns) + else: + col_list = "*" + query = f"SELECT {col_list} FROM {qualified}" + + where_clause = build_where_clause_inline(conditions, quote_char="`") + if where_clause: + query = f"{query} {where_clause}" + + if sort_columns: + direction = "DESC" if sort_order == "desc" else "ASC" + order_cols = ", ".join(f"{_bt(c)} {direction}" for c in sort_columns) + query = f"{query} ORDER BY {order_cols}" + + query = f"{query} LIMIT {int(size)}" + + logger.info("Executing Databricks query: %s...", query[:200]) + arrow_table = self._query_arrow(query) + logger.info("Fetched %d rows from Databricks", arrow_table.num_rows) + return arrow_table diff --git a/py-src/data_formulator/data_loader/kusto_data_loader.py b/py-src/data_formulator/data_loader/kusto_data_loader.py index 3b0ace9d..bf2f1360 100644 --- a/py-src/data_formulator/data_loader/kusto_data_loader.py +++ b/py-src/data_formulator/data_loader/kusto_data_loader.py @@ -1,9 +1,13 @@ import json import logging +import os import re +import time from typing import Any import pandas as pd import pyarrow as pa +import requests as http +from azure.core.credentials import AccessToken # ISO-8601 date / datetime literal (optional time, fractional seconds and # timezone). Values matching this are emitted as KQL ``datetime(...)`` literals @@ -21,6 +25,56 @@ logger = logging.getLogger(__name__) +class _KustoDelegatedCredential: + """Azure TokenCredential backed by an OAuth refresh token.""" + + def __init__( + self, + cluster: str, + access_token: str, + refresh_token: str, + expires_at: float, + ) -> None: + self.cluster = cluster.rstrip("/") + self.access_token = access_token + self.refresh_token = refresh_token + self.expires_at = expires_at + + def get_token(self, *scopes: str, **_kwargs: Any) -> AccessToken: + if self.access_token and self.expires_at > time.time() + 300: + return AccessToken(self.access_token, int(self.expires_at)) + + client_id = os.environ.get("KUSTO_OAUTH_CLIENT_ID", "") + tenant_id = os.environ.get("KUSTO_OAUTH_TENANT_ID", "organizations") + authority = os.environ.get( + "KUSTO_OAUTH_AUTHORITY_HOST", "https://login.microsoftonline.com", + ).rstrip("/") + if not client_id or not self.refresh_token: + raise RuntimeError("Kusto delegated sign-in has expired; sign in again") + + data = { + "client_id": client_id, + "grant_type": "refresh_token", + "refresh_token": self.refresh_token, + "scope": scopes[0] if scopes else f"{self.cluster}/.default", + } + client_secret = os.environ.get("KUSTO_OAUTH_CLIENT_SECRET", "") + if client_secret: + data["client_secret"] = client_secret + response = http.post( + f"{authority}/{tenant_id}/oauth2/v2.0/token", + data=data, + timeout=15, + ) + if not response.ok: + raise RuntimeError("Kusto delegated token refresh failed; sign in again") + tokens = response.json() + self.access_token = tokens["access_token"] + self.refresh_token = tokens.get("refresh_token", self.refresh_token) + self.expires_at = time.time() + tokens.get("expires_in", 3600) + return AccessToken(self.access_token, int(self.expires_at)) + + def _coerce_int(value: Any) -> int | None: """Best-effort conversion of a Kusto stat field to ``int``. @@ -53,7 +107,8 @@ def list_params() -> list[dict[str, Any]]: @classmethod def auth_paths(cls) -> list[dict[str, Any]]: - return [ + microsoft_sign_in = bool(os.environ.get("KUSTO_OAUTH_CLIENT_ID")) + paths = [ { "id": "ambient", "label": "Azure default identity", @@ -61,7 +116,7 @@ def auth_paths(cls) -> list[dict[str, Any]]: "fields": [], "required_fields": [], "kind": "ambient", - "default": True, + "default": not microsoft_sign_in, }, { "id": "service_principal", @@ -72,18 +127,45 @@ def auth_paths(cls) -> list[dict[str, Any]]: "kind": "credentials", }, ] + if microsoft_sign_in: + paths.insert(0, { + "id": "microsoft_sign_in", + "label": "Sign in with Microsoft", + "description": "Use your Microsoft identity and existing Kusto permissions.", + "fields": [], + "required_fields": [], + "kind": "delegated_login", + "default": True, + }) + return paths @classmethod def infer_auth_path(cls, params: dict[str, Any]) -> str: if all(params.get(name) for name in ("client_id", "client_secret", "tenant_id")): return "service_principal" + if params.get("access_token"): + return "microsoft_sign_in" return "ambient" + @staticmethod + def delegated_login_config() -> dict[str, Any] | None: + if not os.environ.get("KUSTO_OAUTH_CLIENT_ID"): + return None + return { + "login_url": "/api/auth/kusto/login", + "label": "Sign in with Microsoft", + "params": ["kusto_cluster"], + } + @staticmethod def auth_instructions() -> str: - return """**Option 1 — Azure Default Identity (recommended):** Leave the auth fields empty. DF connects using the host's ambient Azure credentials — your Azure CLI login (`az login`) when running locally, or a Managed Identity when deployed to Azure. That identity must be granted access to the cluster. + return """**Option 1 — Sign in with Microsoft (recommended):** Sign in as yourself and use your existing Kusto permissions. This option appears when the server has `KUSTO_OAUTH_CLIENT_ID` configured. + + **Option 2 — Azure Default Identity:** Use the host's ambient Azure credentials — your Azure CLI login (`az login`) when running locally, or a Managed Identity when deployed to Azure. + + **Option 3 — Service Principal:** Provide `client_id`, `client_secret`, and `tenant_id` for a service principal with cluster access. -**Option 2 — Service Principal:** Provide `client_id`, `client_secret`, and `tenant_id` for a service principal with cluster access.""" + Every identity must already have data-plane access to the selected Kusto database.""" def __init__(self, params: dict[str, Any]): self.params = params @@ -95,10 +177,11 @@ def __init__(self, params: dict[str, Any]): self.client_secret = params.get("client_secret", None) self.tenant_id = params.get("tenant_id", None) - # Optional delegated user token (Kusto-audience). Reserved for a future - # user-impersonation sign-in; when absent the loader falls through to - # ambient Azure credentials (az login / Managed Identity). + # Optional delegated user token (Kusto-audience). When absent the loader + # falls through to ambient Azure credentials or a service principal. self.access_token = params.get("access_token", None) + self.refresh_token = params.get("refresh_token", None) + self.token_expires_at = float(params.get("token_expires_at") or 0) try: self.client = KustoClient(self._build_kcsb()) @@ -119,9 +202,18 @@ def _build_kcsb(self) -> KustoConnectionStringBuilder: 2. Service principal (``client_id`` / ``client_secret`` / ``tenant_id``) 3. ``DefaultAzureCredential`` (``az login`` / Managed Identity / etc.) """ - # 1. Explicit Kusto user token (already scoped for the cluster). + # 1. Delegated Kusto user token (already scoped for the cluster). if self.access_token: logger.info("Using delegated user token for Kusto client.") + if self.refresh_token: + credential = _KustoDelegatedCredential( + self.kusto_cluster, + self.access_token, + self.refresh_token, + self.token_expires_at, + ) + return KustoConnectionStringBuilder.with_azure_token_credential( + self.kusto_cluster, credential) return KustoConnectionStringBuilder.with_aad_user_token_authentication( self.kusto_cluster, self.access_token) diff --git a/py-src/data_formulator/data_loader/mssql_data_loader.py b/py-src/data_formulator/data_loader/mssql_data_loader.py index 4be57955..8503d4ae 100644 --- a/py-src/data_formulator/data_loader/mssql_data_loader.py +++ b/py-src/data_formulator/data_loader/mssql_data_loader.py @@ -1,6 +1,7 @@ import json import logging import math +import struct from typing import Any import pyarrow as pa @@ -12,6 +13,13 @@ log = logging.getLogger(__name__) +# ODBC connection attribute for passing a pre-fetched Entra ID (Azure AD) +# access token to the SQL Server driver (SQL_COPT_SS_ACCESS_TOKEN). +_SQL_COPT_SS_ACCESS_TOKEN = 1256 + +# Token audience/scope for Azure SQL / SQL Server Entra ID authentication. +_AZURE_SQL_SCOPE = "https://database.windows.net/.default" + def _is_nan(value) -> bool: """Check if a value is NaN (works for float, int, None).""" @@ -52,7 +60,7 @@ def list_params() -> list[dict[str, Any]]: "required": False, "default": "", "tier": "auth", - "description": "Username (leave empty for Windows Authentication)", + "description": "Username (leave empty for Entra ID / Windows auth)", }, { "name": "password", @@ -61,7 +69,7 @@ def list_params() -> list[dict[str, Any]]: "default": "", "sensitive": True, "tier": "auth", - "description": "Password (leave empty for Windows Authentication)", + "description": "Password (leave empty for Entra ID / Windows auth)", }, { "name": "port", @@ -106,20 +114,76 @@ def list_params() -> list[dict[str, Any]]: ] return params_list + @classmethod + def auth_paths(cls) -> list[dict[str, Any]]: + return [ + { + "id": "entra_id", + "label": "Microsoft Entra ID (az login)", + "description": ( + "Run `az login` in your terminal, then connect with no " + "password. Also works with Managed Identity, VS Code, and " + "environment credentials." + ), + "fields": [], + "required_fields": [], + "kind": "ambient", + "default": True, + # In local mode the UI shows an in-app "Sign in with Azure CLI" + # button wired to these endpoints so users can az login without + # leaving the app. + "cli_login": { + "provider": "azure", + "label": "Sign in with Azure CLI", + "status_url": "/api/local/azure-status", + "login_url": "/api/local/azure-login", + }, + }, + { + "id": "sql_auth", + "label": "SQL Server authentication", + "description": "Sign in with a SQL Server username and password.", + "fields": ["user", "password"], + "required_fields": ["user", "password"], + "kind": "credentials", + }, + { + "id": "windows_auth", + "label": "Windows authentication", + "description": "Use the host's Windows identity (Trusted Connection). Windows only.", + "fields": [], + "required_fields": [], + "kind": "ambient", + }, + ] + + @classmethod + def infer_auth_path(cls, params: dict[str, Any]) -> str: + selected = str(params.get("_auth_path") or "").strip() + if selected: + return selected + if params.get("user") and params.get("password"): + return "sql_auth" + return "entra_id" + @staticmethod def auth_instructions() -> str: - return """**Example (SQL auth):** server: `localhost` · database: `mydb` · user: `sa` · password: `MyP@ss` · port: `1433` + return """**Microsoft Entra ID (recommended):** Run `az login` once in your terminal, then start Data Formulator. Choose *Microsoft Entra ID*, fill in only `server` and (optionally) `database`, and leave username/password empty — your Azure CLI credentials are used automatically. Managed Identity, VS Code, and environment credentials also work via `DefaultAzureCredential`. -**Example (Windows auth):** server: `localhost\\SQLEXPRESS` · database: `mydb` (leave user/password empty) +> Your Entra identity must be granted access to the database, e.g. an admin runs `CREATE USER [you@contoso.com] FROM EXTERNAL PROVIDER;` and grants the needed roles. -**Prerequisites (macOS/Linux only):** -Install ODBC driver: `brew install unixodbc msodbcsql17` (macOS) or `sudo apt-get install unixodbc-dev msodbcsql17` (Ubuntu/Debian). Windows usually has these pre-installed. +**Example (Entra ID):** server: `myserver.database.windows.net` · database: `mydb` (username/password empty) + +**SQL Server authentication:** Choose *SQL Server authentication* and provide username and password. -**Authentication:** -- **Windows Auth:** Leave user/password empty (recommended for local dev) -- **SQL Server Auth:** Provide username and password +**Example (SQL auth):** server: `localhost` · database: `mydb` · user: `sa` · password: `MyP@ss` · port: `1433` -**Troubleshooting:** Ensure SQL Server service is running. Verify TCP/IP is enabled in SQL Server Configuration Manager. Test with `sqlcmd -S -d -U -P `.""" +**Windows authentication (Windows only):** Choose *Windows authentication* and leave username/password empty. + +**Prerequisites (macOS/Linux only):** +Install the ODBC driver: `brew install unixodbc msodbcsql18` (macOS) or `sudo apt-get install unixodbc-dev msodbcsql18` (Ubuntu/Debian). For Entra ID you also need the Azure CLI (`brew install azure-cli`) and to run `az login`. + +**Troubleshooting:** Confirm you are signed in with `az account show`. Ensure the SQL Server service is running and TCP/IP is enabled. Test SQL auth with `sqlcmd -S -d -U -P `.""" def __init__(self, params: dict[str, Any]): from data_formulator.security.log_sanitizer import sanitize_params @@ -137,10 +201,12 @@ def __init__(self, params: dict[str, Any]): self.trust_server_certificate = params.get("trust_server_certificate", "no") self.connection_timeout = params.get("connection_timeout", "30") + self.auth_path = params.get("_auth_path") or self.infer_auth_path(params) + # When no database specified, connect to master for catalog browsing connect_db = self.database or "master" - # Build ODBC connection string + # Build the auth-independent part of the ODBC connection string. conn_str = ( f"DRIVER={{{self.driver}}};" f"SERVER={self.server},{self.port};" @@ -149,18 +215,57 @@ def __init__(self, params: dict[str, Any]): f"TrustServerCertificate={self.trust_server_certificate};" f"Connection Timeout={self.connection_timeout};" ) - if self.user: + + connect_kwargs: dict[str, Any] = {} + if self.auth_path == "entra_id": + # Entra ID: fetch a token via DefaultAzureCredential (az login / + # Managed Identity / VS Code / env) and hand it to the driver. + # No UID/PWD/Trusted_Connection goes into the string. + connect_kwargs["attrs_before"] = { + _SQL_COPT_SS_ACCESS_TOKEN: self._get_entra_id_token_struct() + } + elif self.user or self.auth_path == "sql_auth": conn_str += f"UID={self.user};PWD={self.password};" else: conn_str += "Trusted_Connection=yes;" try: - self._conn = pyodbc.connect(conn_str) + self._conn = pyodbc.connect(conn_str, **connect_kwargs) log.info(f"Successfully connected to SQL Server: {self.server}/{self.database}") except Exception as e: log.error(f"Failed to connect to SQL Server: {e}") raise ValueError(f"Failed to connect to SQL Server '{self.server}': {e}") from e + @staticmethod + def _get_entra_id_token_struct() -> bytes: + """Fetch an Entra ID access token for Azure SQL and pack it for ODBC. + + Uses ``DefaultAzureCredential`` so a local ``az login`` (or Managed + Identity / VS Code / environment credentials) is enough to connect. + The token is packed into the little-endian, length-prefixed UTF-16-LE + struct expected by the SQL Server ODBC access-token attribute. + """ + try: + from azure.identity import DefaultAzureCredential + except ImportError as e: + raise ValueError( + "Microsoft Entra ID authentication requires the 'azure-identity' " + "package. Install it with `uv pip install azure-identity`." + ) from e + + try: + credential = DefaultAzureCredential() + token = credential.get_token(_AZURE_SQL_SCOPE).token + except Exception as e: + raise ValueError( + "Failed to acquire a Microsoft Entra ID token. Run `az login` in " + "your terminal (or configure a Managed Identity), then retry. " + f"Details: {e}" + ) from e + + token_bytes = token.encode("UTF-16-LE") + return struct.pack(f"=2.8.0", # OIDC JWT verification (includes cryptography) "requests", # GitHub OAuth code exchange, Superset API calls diff --git a/pytest.ini b/pytest.ini index 209f4201..52048650 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,3 +12,4 @@ markers = auth: authentication provider tests vault: credential vault tests xfail_known_bug: regression tests for known bugs + live: end-to-end tests that hit a real external service (skipped unless credentials are provided via env) diff --git a/requirements.txt b/requirements.txt index c273db1d..a0478fb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,6 +34,7 @@ boto3 google-cloud-bigquery google-auth db-dtypes +databricks-sql-connector # SSO / Auth deps PyJWT[crypto]>=2.8.0 diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 33914c3c..bface3f7 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -80,7 +80,7 @@ export interface ServerConfig { effective_hierarchy: Array<{key: string; label: string}>; auth_instructions: string; auth_mode?: string; - delegated_login?: { login_url: string; label?: string } | null; + delegated_login?: { login_url: string; label?: string; params?: string[] } | null; }>; DISABLED_SOURCES?: Record; CONNECTED_CONNECTORS?: string[]; diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index 103c9a5d..d550a273 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -572,6 +572,13 @@ export interface ConnectorAuthPath { required_fields?: string[]; kind: 'credentials' | 'ambient' | 'delegated_login' | 'token_exchange'; default?: boolean; + /** Optional in-app CLI sign-in (local mode only), e.g. `az login`. */ + cli_login?: { + provider: string; + label: string; + status_url: string; + login_url: string; + }; } /** A registered connector instance from GET /api/connectors */ @@ -595,5 +602,5 @@ export interface ConnectorInstance { auth_mode?: string; auth_paths?: ConnectorAuthPath[]; auth_instructions?: string; - delegated_login?: { login_url: string; label?: string } | null; + delegated_login?: { login_url: string; label?: string; params?: string[] } | null; } diff --git a/src/components/ConnectorFormCard.tsx b/src/components/ConnectorFormCard.tsx index 26469dbd..df55e19d 100644 --- a/src/components/ConnectorFormCard.tsx +++ b/src/components/ConnectorFormCard.tsx @@ -36,7 +36,7 @@ interface LoaderMeta { auth_mode?: string; auth_paths?: ConnectorAuthPath[]; auth_instructions?: string; - delegated_login?: { login_url: string; label?: string } | null; + delegated_login?: { login_url: string; label?: string; params?: string[] } | null; } interface ConnectorFormCardProps { diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 86c12c61..77d51ff3 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -345,8 +345,16 @@ "datasets": "Datasets", "dashboards": "Dashboards", "rememberCredentials": "Remember credentials", + "setupDetails": "Setup details", + "setupFieldsIntro": "Provide the following to connect:", + "optional": "optional", "connectionTimeout": "Connection timed out. Please check your credentials and try again.", "delegatedLogin": "Login via service", + "cliLoginInProgress": "Signing in…", + "cliLoginSwitch": "Switch account", + "cliLoginSignedInAs": "Signed in as {{user}}", + "cliNotInstalled": "Azure CLI not found — install it or sign in via a terminal.", + "cliLoginFailed": "Sign-in failed. Try running the login command in a terminal.", "popupBlocked": "Popup was blocked. Please allow popups and try again.", "tierConnection": "Connection", "tierAuth": "Sign in", diff --git a/src/i18n/locales/en/loader.json b/src/i18n/locales/en/loader.json index 6376c779..42f09667 100644 --- a/src/i18n/locales/en/loader.json +++ b/src/i18n/locales/en/loader.json @@ -11,14 +11,14 @@ "mssql": { "server": "SQL Server host address or instance name", "database": "Database name (leave empty to browse all databases)", - "user": "Username (leave empty for Windows Authentication)", - "password": "Password (leave empty for Windows Authentication)", + "user": "Username (leave empty for Entra ID / Windows auth)", + "password": "Password (leave empty for Entra ID / Windows auth)", "port": "SQL Server port (default: 1433)", "driver": "ODBC driver name", "encrypt": "Enable encryption (yes/no)", "trust_server_certificate": "Trust server certificate (yes/no)", "connection_timeout": "Connection timeout in seconds", - "authInstructions": "**Example (SQL auth):** server: `localhost` · database: `mydb` · user: `sa` · password: `MyP@ss` · port: `1433`\n\n**Example (Windows auth):** server: `localhost\\SQLEXPRESS` · database: `mydb` (leave user/password empty)\n\n**Prerequisites (macOS/Linux only):**\nInstall ODBC driver: `brew install unixodbc msodbcsql17` (macOS) or `sudo apt-get install unixodbc-dev msodbcsql17` (Ubuntu/Debian). Windows usually has these pre-installed.\n\n**Authentication:**\n- **Windows Auth:** Leave user/password empty (recommended for local dev)\n- **SQL Server Auth:** Provide username and password\n\n**Troubleshooting:** Ensure SQL Server service is running. Verify TCP/IP is enabled in SQL Server Configuration Manager. Test with `sqlcmd -S -d -U -P `." + "authInstructions": "**Microsoft Entra ID (recommended):** Run `az login` once in your terminal, then start Data Formulator. Choose *Microsoft Entra ID*, fill in only `server` and (optionally) `database`, and leave username/password empty — your Azure CLI credentials are used automatically. Managed Identity, VS Code, and environment credentials also work via `DefaultAzureCredential`.\n\n> Your Entra identity must be granted access to the database, e.g. an admin runs `CREATE USER [you@contoso.com] FROM EXTERNAL PROVIDER;` and grants the needed roles.\n\n**Example (Entra ID):** server: `myserver.database.windows.net` · database: `mydb` (username/password empty)\n\n**SQL Server authentication:** Choose *SQL Server authentication* and provide username and password.\n\n**Example (SQL auth):** server: `localhost` · database: `mydb` · user: `sa` · password: `MyP@ss` · port: `1433`\n\n**Windows authentication (Windows only):** Choose *Windows authentication* and leave username/password empty.\n\n**Prerequisites (macOS/Linux only):**\nInstall the ODBC driver: `brew install unixodbc msodbcsql18` (macOS) or `sudo apt-get install unixodbc-dev msodbcsql18` (Ubuntu/Debian). For Entra ID you also need the Azure CLI (`brew install azure-cli`) and to run `az login`.\n\n**Troubleshooting:** Confirm you are signed in with `az account show`. Ensure the SQL Server service is running and TCP/IP is enabled. Test SQL auth with `sqlcmd -S -d -U -P `." }, "postgresql": { "user": "PostgreSQL username", @@ -70,7 +70,15 @@ "client_id": "Service principal only", "client_secret": "Service principal only", "tenant_id": "Service principal only", - "authInstructions": "**Option 1 — Azure Default Identity (easiest):** Leave auth fields empty. DF will automatically use your Azure CLI login (`az login`), Managed Identity, VS Code credentials, or environment variables — whichever is available.\n\n**Option 2 — Service Principal:** Provide `client_id`, `client_secret`, and `tenant_id` for a service principal with cluster access." + "authInstructions": "**Option 1 — Sign in with Microsoft (recommended):** Sign in as yourself and use your existing Kusto permissions. This option appears when the server has `KUSTO_OAUTH_CLIENT_ID` configured.\n\n**Option 2 — Azure Default Identity:** Use your Azure CLI login (`az login`), Managed Identity, VS Code credentials, or environment credentials.\n\n**Option 3 — Service Principal:** Provide `client_id`, `client_secret`, and `tenant_id` for a service principal with cluster access.\n\nEvery identity must already have data-plane access to the selected Kusto database." + }, + "databricks": { + "server_hostname": "e.g., adb-1234567890.11.azuredatabricks.net", + "http_path": "SQL warehouse HTTP path, e.g., /sql/1.0/warehouses/abc123", + "catalog": "Unity Catalog name (leave empty to browse all catalogs)", + "schema": "Schema name (leave empty to browse all schemas in the catalog)", + "access_token": "Databricks personal access token (dapi...)", + "authInstructions": "**Where to find these:** In your Databricks workspace, open **SQL → SQL Warehouses** (left sidebar), click your warehouse, and open the **Connection details** tab — copy **Server hostname** and **HTTP path** from there.\n\n**Access token:** Click your avatar (top-right) → **Settings → Developer → Access tokens → Generate new token**. It starts with `dapi` and is shown only once.\n\n**Permissions:** The token's user needs `USE CATALOG` / `USE SCHEMA` and `SELECT` on the Unity Catalog objects you want to read.\n\n**Scope:** Leave *catalog* and *schema* empty to browse everything you can access, or set them to jump straight to a specific catalog/schema — e.g. try the built-in `samples` catalog → `nyctaxi` → `trips`.\n\n**No account?** Databricks Free Edition is serverless, free, and ships the `samples` catalog — no cluster or warehouse setup needed." }, "superset": { "url": "Superset base URL (e.g. https://bi.company.com)", diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index d99518ed..10404e65 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -345,8 +345,16 @@ "datasets": "数据集", "dashboards": "仪表盘", "rememberCredentials": "记住凭据", + "setupDetails": "设置详情", + "setupFieldsIntro": "请填写以下信息以建立连接：", + "optional": "可选", "connectionTimeout": "连接超时。请检查凭据后重试。", "delegatedLogin": "通过服务登录", + "cliLoginInProgress": "正在登录…", + "cliLoginSwitch": "切换账户", + "cliLoginSignedInAs": "已登录：{{user}}", + "cliNotInstalled": "未找到 Azure CLI — 请安装或在终端中登录。", + "cliLoginFailed": "登录失败。请尝试在终端中运行登录命令。", "popupBlocked": "弹出窗口被阻止。请允许弹出窗口后重试。", "tierConnection": "连接", "tierAuth": "登录", diff --git a/src/i18n/locales/zh/loader.json b/src/i18n/locales/zh/loader.json index 6c004f5e..e4d815bf 100644 --- a/src/i18n/locales/zh/loader.json +++ b/src/i18n/locales/zh/loader.json @@ -11,14 +11,14 @@ "mssql": { "server": "SQL Server 主机地址或实例名", "database": "数据库名称（留空可浏览所有数据库）", - "user": "用户名（留空使用 Windows 认证）", - "password": "密码（留空使用 Windows 认证）", + "user": "用户名（使用 Entra ID / Windows 认证时留空）", + "password": "密码（使用 Entra ID / Windows 认证时留空）", "port": "SQL Server 端口（默认 1433）", "driver": "ODBC 驱动名称", "encrypt": "启用加密（yes/no）", "trust_server_certificate": "信任服务器证书（yes/no）", "connection_timeout": "连接超时时间（秒）", - "authInstructions": "**示例（SQL 认证）：** server: `localhost` · database: `mydb` · user: `sa` · password: `MyP@ss` · port: `1433`\n\n**示例（Windows 认证）：** server: `localhost\\SQLEXPRESS` · database: `mydb`（用户名和密码留空）\n\n**前置条件（仅 macOS/Linux）：**\n安装 ODBC 驱动：`brew install unixodbc msodbcsql17`（macOS）或 `sudo apt-get install unixodbc-dev msodbcsql17`（Ubuntu/Debian）。Windows 通常已预装。\n\n**认证方式：**\n- **Windows 认证：** 用户名和密码留空（推荐用于本地开发）\n- **SQL Server 认证：** 提供用户名和密码\n\n**排查：** 确保 SQL Server 服务正在运行。在 SQL Server 配置管理器中确认 TCP/IP 已启用。用 `sqlcmd -S -d -U -P ` 测试连接。" + "authInstructions": "**Microsoft Entra ID（推荐）：** 先在终端运行一次 `az login`，然后启动 Data Formulator。选择 *Microsoft Entra ID*，只填写 `server`（和可选的 `database`），用户名/密码留空 — 会自动使用你的 Azure CLI 凭据。也支持通过 `DefaultAzureCredential` 使用托管标识、VS Code 和环境凭据。\n\n> 你的 Entra 身份必须已获授权访问该数据库，例如管理员运行 `CREATE USER [you@contoso.com] FROM EXTERNAL PROVIDER;` 并授予相应角色。\n\n**示例（Entra ID）：** server: `myserver.database.windows.net` · database: `mydb`（用户名/密码留空）\n\n**SQL Server 认证：** 选择 *SQL Server 认证* 并提供用户名和密码。\n\n**示例（SQL 认证）：** server: `localhost` · database: `mydb` · user: `sa` · password: `MyP@ss` · port: `1433`\n\n**Windows 认证（仅 Windows）：** 选择 *Windows 认证* 并将用户名/密码留空。\n\n**前置条件（仅 macOS/Linux）：**\n安装 ODBC 驱动：`brew install unixodbc msodbcsql18`（macOS）或 `sudo apt-get install unixodbc-dev msodbcsql18`（Ubuntu/Debian）。使用 Entra ID 还需安装 Azure CLI（`brew install azure-cli`）并运行 `az login`。\n\n**排查：** 用 `az account show` 确认已登录。确保 SQL Server 服务正在运行且 TCP/IP 已启用。用 `sqlcmd -S -d -U -P ` 测试 SQL 认证。" }, "postgresql": { "user": "PostgreSQL 用户名", @@ -70,7 +70,15 @@ "client_id": "仅服务主体", "client_secret": "仅服务主体", "tenant_id": "仅服务主体", - "authInstructions": "**方式 1 — Azure 默认身份（最简单）：** 认证字段留空。DF 会自动使用 Azure CLI 登录（`az login`）、托管标识、VS Code 凭据或环境变量 — 按顺序尝试。\n\n**方式 2 — 服务主体：** 提供具有集群访问权限的 `client_id`、`client_secret` 和 `tenant_id`。" + "authInstructions": "**方式 1 — 使用 Microsoft 登录（推荐）：** 使用自己的身份登录并沿用现有 Kusto 权限。服务器配置 `KUSTO_OAUTH_CLIENT_ID` 后会显示此选项。\n\n**方式 2 — Azure 默认身份：** 使用 Azure CLI 登录（`az login`）、托管标识、VS Code 凭据或环境凭据。\n\n**方式 3 — 服务主体：** 提供具有集群访问权限的 `client_id`、`client_secret` 和 `tenant_id`。\n\n无论选择哪种身份，都必须预先拥有所选 Kusto 数据库的数据平面访问权限。" + }, + "databricks": { + "server_hostname": "例如 adb-1234567890.11.azuredatabricks.net", + "http_path": "SQL 仓库 HTTP 路径，例如 /sql/1.0/warehouses/abc123", + "catalog": "Unity Catalog 名称（留空则浏览所有 catalog）", + "schema": "架构名称（留空则浏览该 catalog 下所有架构）", + "access_token": "Databricks 个人访问令牌（dapi...）", + "authInstructions": "**在哪里找到这些值：** 在 Databricks 工作区中打开 **SQL → SQL Warehouses**（左侧栏），点击你的仓库，进入 **Connection details** 选项卡，从中复制 **Server hostname** 和 **HTTP path**。\n\n**访问令牌：** 点击右上角头像 → **Settings → Developer → Access tokens → Generate new token**，令牌以 `dapi` 开头且仅显示一次。\n\n**权限：** 该令牌所属用户需要对目标 Unity Catalog 对象拥有 `USE CATALOG` / `USE SCHEMA` 和 `SELECT` 权限。\n\n**范围：** 将 *catalog* 和 *schema* 留空可浏览你有权访问的全部内容，或填写以直接进入指定的 catalog/架构 —— 例如可试用内置的 `samples` catalog → `nyctaxi` → `trips`。\n\n**没有账号？** Databricks Free Edition 免费、无服务器，并内置 `samples` catalog，无需配置集群或仓库。" }, "superset": { "url": "Superset 基础 URL（如 https://bi.company.com）", diff --git a/src/icons.tsx b/src/icons.tsx index ea86635c..73f0662e 100644 --- a/src/icons.tsx +++ b/src/icons.tsx @@ -94,6 +94,7 @@ const CONNECTOR_ICON_MAP: Record> = { bigquery: QueryEngineIcon, kusto: QueryEngineIcon, athena: QueryEngineIcon, + databricks: QueryEngineIcon, // BI / dashboards superset: DashboardIcon, // Local @@ -127,6 +128,7 @@ const CONNECTOR_CATEGORY_ORDER: Record = { bigquery: 3, BigQueryDataLoader: 3, kusto: 3, KustoDataLoader: 3, athena: 3, AthenaDataLoader: 3, + databricks: 3, DatabricksDataLoader: 3, // Dashboard superset: 4, SupersetLoader: 4, }; diff --git a/src/views/DBTableManager.tsx b/src/views/DBTableManager.tsx index e3747aeb..c3714455 100644 --- a/src/views/DBTableManager.tsx +++ b/src/views/DBTableManager.tsx @@ -12,10 +12,11 @@ import { IconButton, Tooltip, Autocomplete, - Radio, - RadioGroup, + ToggleButton, + ToggleButtonGroup, } from '@mui/material'; import OpenInNewIcon from '@mui/icons-material/OpenInNew'; +import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; import { CONNECTOR_ACTION_URLS } from '../app/utils'; import { apiRequest, type ApiError } from '../app/apiClient'; @@ -85,7 +86,7 @@ export const DataLoaderForm: React.FC<{ autoConnect?: boolean, /** When true, attempt SSO token passthrough on mount (no popup). */ ssoAutoConnect?: boolean, - delegatedLogin?: { login_url: string; label?: string } | null, + delegatedLogin?: { login_url: string; label?: string; params?: string[] } | null, authMode?: string, authPaths?: ConnectorAuthPath[], connectionName?: { @@ -141,10 +142,36 @@ export const DataLoaderForm: React.FC<{ const localizedAuthInstructions = t(`loader.${loaderTypeKey}.authInstructions`, { defaultValue: authInstructions.trim(), }); + // Field-level help text, without the ••••• masking that getParamPlaceholder + // applies to stored secrets — used to explain each field in Setup details. + const getParamHelp = (paramDef: {name: string; default?: string | number | boolean; description?: string}) => { + const fallback = paramDef.description || ''; + return t(`loader.${loaderTypeKey}.${paramDef.name}`, { + defaultValue: t(`loader._common.${paramDef.name}`, { defaultValue: fallback }), + }); + }; + // Setup details always shows something actionable: prefer the connector's + // authored guidance (concrete steps), otherwise auto-explain the fields the + // user has to fill in so they know what each one expects. + const fieldGuide = paramDefs + .filter((p) => p.tier !== 'auth') + .map((p) => { + const help = getParamHelp(p); + const optional = p.required + ? '' + : ` _(${t('db.optional', { defaultValue: 'optional' })})_`; + return `- **${p.name}**${optional}${help ? ` — ${help}` : ''}`; + }) + .join('\n'); + const setupDetailsContent = localizedAuthInstructions + || (fieldGuide + ? `${t('db.setupFieldsIntro', { defaultValue: 'Provide the following to connect:' })}\n\n${fieldGuide}` + : ''); // Effective connectorId — may be updated by onBeforeConnect (e.g. AddConnectionPanel) const connectorIdRef = useRef(connectorId); useEffect(() => { connectorIdRef.current = connectorId; }, [connectorId]); const params = useSelector((state: DataFormulatorState) => state.dataLoaderConnectParams[dataLoaderType] ?? {}); + const isLocalMode = useSelector((state: DataFormulatorState) => !!state.serverConfig?.IS_LOCAL_MODE); // Materialize declared defaults and the default authentication path as // actual form values rather than placeholders. Existing user-entered or @@ -171,7 +198,6 @@ export const DataLoaderForm: React.FC<{ let [isConnecting, setIsConnecting] = useState(false); const [persistCredentials, setPersistCredentials] = useState(true); - // High-level progress shown while connecting (e.g. Kusto reporting which // database it's currently listing). Polled from the backend during the // connect request; cleared when it resolves. @@ -181,6 +207,60 @@ export const DataLoaderForm: React.FC<{ const [databaseDiscoveryError, setDatabaseDiscoveryError] = useState(''); const [databaseMenuOpen, setDatabaseMenuOpen] = useState(false); + // In-app CLI sign-in (local mode only), e.g. `az login` for Entra ID. + const [cliLoginStatus, setCliLoginStatus] = useState<{ installed: boolean; signed_in: boolean; account: { user?: string } | null } | null>(null); + const [cliLoginBusy, setCliLoginBusy] = useState(false); + const [cliLoginError, setCliLoginError] = useState(''); + + // The auth path the user has currently selected (also computed in the + // render body; duplicated here so effects/handlers can react to it). + const activeAuthPath = authPaths.find(path => path.id === params._auth_path) + || authPaths.find(path => path.default) + || authPaths[0]; + const cliLogin = (isLocalMode && activeAuthPath?.cli_login) ? activeAuthPath.cli_login : undefined; + const cliStatusUrl = cliLogin?.status_url; + + // Fetch current CLI sign-in status when a CLI-login auth path is selected. + useEffect(() => { + if (!cliStatusUrl) { setCliLoginStatus(null); setCliLoginError(''); return; } + let cancelled = false; + (async () => { + try { + const { data } = await apiRequest(cliStatusUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + if (!cancelled) setCliLoginStatus(data); + } catch { + if (!cancelled) setCliLoginStatus(null); + } + })(); + return () => { cancelled = true; }; + }, [cliStatusUrl]); + + const handleCliLogin = useCallback(async () => { + if (!cliLogin) return; + setCliLoginBusy(true); + setCliLoginError(''); + try { + const { data } = await apiRequest(cliLogin.login_url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + setCliLoginStatus({ installed: true, signed_in: !!data.signed_in, account: data.account ?? null }); + } catch (error: any) { + setCliLoginError( + error?.apiError?.message + || error?.message + || t('db.cliLoginFailed', { defaultValue: 'Sign-in failed. Try running the login command in a terminal.' }), + ); + } finally { + setCliLoginBusy(false); + } + }, [cliLogin, t]); + // Sensitive params (passwords, tokens, secrets) live in component state only — // never persisted to Redux / localStorage. // Sensitivity is declared by the loader via `sensitive: true` or `type: "password"`. @@ -350,9 +430,14 @@ export const DataLoaderForm: React.FC<{ const url = new URL(delegatedLogin.login_url, window.location.origin); url.searchParams.set('df_origin', window.location.origin); - // Pass auth-tier form params (e.g. client_id, tenant_id) to the login endpoint + // Pass only fields explicitly requested by the login config. Legacy + // delegated connectors default to their non-sensitive auth fields. + const loginParamNames = new Set( + delegatedLogin.params + || paramDefs.filter(p => p.tier === 'auth' && !p.sensitive && p.type !== 'password').map(p => p.name), + ); for (const p of paramDefs) { - if (p.tier === 'auth' && !p.sensitive && p.type !== 'password' && currentParams[p.name]) { + if (loginParamNames.has(p.name) && !p.sensitive && p.type !== 'password' && currentParams[p.name]) { url.searchParams.set(p.name, currentParams[p.name]); } } @@ -374,12 +459,17 @@ export const DataLoaderForm: React.FC<{ } const handler = async (event: MessageEvent) => { - if (event.data?.type !== 'df-sso-auth') return; + if (event.source !== popup || event.data?.type !== 'df-sso-auth') return; window.removeEventListener('message', handler); if (pollTimerRef.current) { clearInterval(pollTimerRef.current); pollTimerRef.current = null; } popup.close(); - const { access_token, refresh_token, user } = event.data; + const { access_token, refresh_token, expires_in, user, error } = event.data; + if (error) { + onFinish("error", error); + setIsConnecting(false); + return; + } if (access_token) { try { // Persist token in TokenStore for Agent and future requests @@ -390,6 +480,7 @@ export const DataLoaderForm: React.FC<{ system_id: connectorIdRef.current, access_token, refresh_token, + expires_in, user, }), }).catch(() => {}); @@ -403,6 +494,7 @@ export const DataLoaderForm: React.FC<{ mode: 'token', access_token, refresh_token, + expires_in, user, params: getCurrentParams(), // include any filled-in params (e.g. url) persist: persistCredentials, @@ -491,12 +583,12 @@ export const DataLoaderForm: React.FC<{ re-auth only. */} <> {formTitle && ( - + {formTitle} )} {!onBeforeConnect && ( - + {dataLoaderType} )} @@ -550,7 +642,7 @@ export const DataLoaderForm: React.FC<{ {title && ( - + {title} )} @@ -558,13 +650,23 @@ export const DataLoaderForm: React.FC<{ ); - // Keep Material UI's native colors, spacing, focus, - // label, and placeholder treatments. Only compact the - // form copy to the surrounding 12px type scale. + const formTextSx = { + fontSize: 12, + lineHeight: 1.5, + fontWeight: 400, + letterSpacing: 0, + }; + const secondaryTextSx = { + ...formTextSx, + color: 'text.secondary', + }; + // Typical Data Formulator body size (12px). Fields, labels + // and placeholders all sit on this one scale. const inputSx = { '& .MuiInputBase-root': { fontSize: 12 }, '& .MuiInputBase-input': { fontSize: 12 }, - '& .MuiInputBase-input::placeholder': { fontSize: 12 }, + '& .MuiInputLabel-root': { fontSize: 12 }, + '& .MuiFormHelperText-root': { fontSize: 11, mx: 0 }, }; const labelShrinkSlotProps = { inputLabel: { shrink: true } }; const paramGridSx = { @@ -609,12 +711,33 @@ export const DataLoaderForm: React.FC<{ loaderTypeKey === 'kusto' && name === 'kusto_cluster'; const isKustoDatabase = (name: string) => loaderTypeKey === 'kusto' && name === 'kusto_database'; + // Left label, right input box. The per-field hint lives + // inside the box as its placeholder, so each row stays a + // single clean line: "name [ value / hint ]". + const renderFieldRow = (paramDef: typeof tierParams[number], input: React.ReactNode) => ( + + + {paramDef.name}{paramDef.required ? ' *' : ''} + + {input} + + ); return ( - + {tierParams.map((paramDef) => ( isKustoCluster(paramDef.name) ? ( + renderFieldRow(paramDef, + - {inputParams.InputProps.endAdornment} - - - - - - - ), - }} + placeholder={getParamHelp(paramDef) || getParamPlaceholder(paramDef)} /> )} slotProps={{ @@ -670,9 +777,16 @@ export const DataLoaderForm: React.FC<{ }, }} /> + + + + + + + ) ) : isKustoDatabase(paramDef.name) ? ( + renderFieldRow(paramDef, { @@ -717,12 +831,9 @@ export const DataLoaderForm: React.FC<{ {...inputParams} sx={inputSx} variant="standard" size="small" fullWidth - slotProps={labelShrinkSlotProps} - label={paramDef.name} - required={paramDef.required} - placeholder={getParamPlaceholder(paramDef)} + placeholder={getParamHelp(paramDef) || getParamPlaceholder(paramDef)} error={!!databaseDiscoveryError} - helperText={databaseDiscoveryError} + helperText={databaseDiscoveryError || undefined} /> )} slotProps={{ @@ -735,21 +846,20 @@ export const DataLoaderForm: React.FC<{ }, }} /> + ) ) : ( + renderFieldRow(paramDef, updateParamDraft(paramDef.name, value)} onCommit={(value) => commitParamDraft(paramDef.name, value)} /> ) + ) ))} ); @@ -763,81 +873,173 @@ export const DataLoaderForm: React.FC<{ || authPaths[0]; const selectedAuthFieldNames = new Set(selectedAuthPath?.fields || authParams.map(p => p.name)); const selectedAuthParams = authParams.filter(p => selectedAuthFieldNames.has(p.name)); - const hasDelegated = !!delegatedLogin?.login_url; + const hasDelegated = !!delegatedLogin?.login_url + && (!selectedAuthPath || selectedAuthPath.kind === 'delegated_login'); const connectLabel = onBeforeConnect ? t('db.createConnector', { defaultValue: 'Create Connector' }) : t('db.connect', { suffix: (params.table_filter || '').trim() ? t('db.withFilter') : '' }); let stepNumber = 0; - const nameStep = connectionName ? ++stepNumber : 0; - const connectionStep = connectionParams.length > 0 ? ++stepNumber : 0; - const authStep = ++stepNumber; + const connectionStep = connectionName || connectionParams.length > 0 ? ++stepNumber : 0; const scopeStep = filterParams.length > 0 ? ++stepNumber : 0; - const actionStep = ++stepNumber; + const authStep = ++stepNumber; return ( - {connectionName && renderTimelineStep( - nameStep, - null, - - - , - )} - - {/* Tier 1: Connection */} - {connectionParams.length > 0 && ( + {/* Connection identity and source coordinates belong together. */} + {(connectionName || connectionParams.length > 0) && ( renderTimelineStep( connectionStep, t('db.tierConnection'), - renderParamGrid(connectionParams), + + {connectionName && ( + + + {connectionName.label} + + + + + + )} + {connectionParams.length > 0 && renderParamGrid(connectionParams)} + , + ) + )} + + {/* Tier 2: connection scope and catalog filters. */} + {filterParams.length > 0 && ( + renderTimelineStep( + scopeStep, + t('db.tierFilter'), + renderParamGrid(filterParams), ) )} - {/* Tier 2: choose an authentication path, then + {/* Final tier: choose an authentication path, then reveal only that path's credential fields. */} {renderTimelineStep( authStep, t('db.tierAuth'), <> {authPaths.length > 1 && ( - dispatch(dfActions.updateDataLoaderConnectParam({ - dataLoaderType, - paramName: '_auth_path', - paramValue: event.target.value, - }))} - sx={{ gap: 1, mb: selectedAuthParams.length > 0 ? 1 : 0 }} + onChange={(_event, value) => { + if (!value) return; + dispatch(dfActions.updateDataLoaderConnectParam({ + dataLoaderType, + paramName: '_auth_path', + paramValue: value, + })); + }} + aria-label={t('db.tierAuth')} + sx={{ + display: 'inline-flex', + '& .MuiToggleButton-root': { + height: 30, + px: 1.5, + py: 0, + ...formTextSx, + textTransform: 'none', + color: 'text.secondary', + borderColor: 'divider', + '&.Mui-selected': { + color: 'primary.main', + bgcolor: 'action.selected', + }, + }, + '& .MuiToggleButtonGroup-grouped': { + borderRadius: 0, + '&:first-of-type': { + borderTopLeftRadius: 4, + borderBottomLeftRadius: 4, + }, + '&:last-of-type': { + borderTopRightRadius: 4, + borderBottomRightRadius: 4, + }, + }, + }} > {authPaths.map(path => ( - } - label={( - {path.label} - )} - sx={{ mr: 2 }} - /> + > + {path.label} + ))} - + )} {authPaths.length > 1 && selectedAuthPath?.description && ( - 0 ? 0.5 : 0 }}> - {selectedAuthPath.description} - + 0 ? 2 : 0, + px: 1, + py: 0.75, + borderRadius: 1, + bgcolor: 'action.hover', + }}> + + + {selectedAuthPath.description} + + + )} + + {isLocalMode && selectedAuthPath?.cli_login && ( + 0 ? 2 : 0 }}> + + + {cliLoginStatus?.signed_in && cliLoginStatus.account?.user && ( + + {t('db.cliLoginSignedInAs', { user: cliLoginStatus.account.user, defaultValue: 'Signed in as {{user}}' })} + + )} + {cliLoginStatus && !cliLoginStatus.installed && ( + + {t('db.cliNotInstalled', { defaultValue: 'Azure CLI not found — install it or sign in via a terminal.' })} + + )} + + {cliLoginError && ( + {cliLoginError} + )} + )} {hasDelegated && selectedAuthParams.length > 0 ? ( @@ -858,29 +1060,20 @@ export const DataLoaderForm: React.FC<{ {/* Right: credential fields + connect */} - - {selectedAuthParams.map((paramDef) => ( - updateParamDraft(paramDef.name, value)} - onCommit={(value) => commitParamDraft(paramDef.name, value)} - /> - ))} - + {renderParamGrid(selectedAuthParams)} ) : hasDelegated ? ( /* Delegated only */ + )} + , + true, )} - {/* Primary submit action is separate from all - parameter sections. Delegated-only sources - complete through their sign-in button. */} - {renderTimelineStep( - actionStep, - null, - - {(!hasDelegated || selectedAuthParams.length > 0) && ( - - )} - {paramDefs.length > 0 && ( - setPersistCredentials(event.target.checked)} - sx={{ p: 0.5 }} - /> - )} - label={( - - {t('db.rememberCredentials')} - - )} - /> - )} - , - true, + {paramDefs.length > 0 && ( + + setPersistCredentials(event.target.checked)} + sx={{ p: 0 }} + /> + )} + label={( + + {t('db.rememberCredentials')} + + )} + /> + )} ); })()} - {localizedAuthInstructions && !hideInstructions && ( - ({ - mt: 3, px: 1.5, py: 1, - backgroundColor: 'rgba(0,0,0,0.02)', - borderRadius: 1, - border: '1px solid rgba(0,0,0,0.06)', - fontFamily: theme.typography.fontFamily, - fontSize: '11px', - color: 'text.secondary', - lineHeight: 1.6, - '& *': { fontFamily: theme.typography.fontFamily, fontSize: 'inherit', lineHeight: 'inherit', color: 'inherit' }, - '& p': { margin: '0 0 4px 0', '&:last-child': { marginBottom: 0 } }, - '& code': { fontSize: '10px', fontFamily: 'monospace !important', backgroundColor: 'rgba(0,0,0,0.06)', padding: '1px 4px', borderRadius: '3px' }, - '& pre': { fontSize: '10px', fontFamily: 'monospace !important', backgroundColor: 'rgba(0,0,0,0.04)', padding: '8px', borderRadius: '4px', overflow: 'auto', margin: '4px 0', '& code': { backgroundColor: 'transparent', padding: 0 } }, - '& a': { color: 'primary.main' }, - '& ul, & ol': { paddingLeft: '20px', margin: '4px 0' }, - '& li': { marginBottom: '2px' }, - '& strong': { fontWeight: 600, color: 'text.primary' }, - '& h1, & h2, & h3, & h4': { fontSize: '12px', fontWeight: 600, color: 'text.primary', margin: '4px 0' }, - })}> - {localizedAuthInstructions} + {setupDetailsContent && ( + + + {t('db.setupDetails', { defaultValue: 'Setup details' })} + + ({ + maxWidth: 720, + fontFamily: theme.typography.fontFamily, + fontSize: 11.5, + lineHeight: 1.5, + color: 'text.secondary', + '& *': { fontSize: 'inherit', lineHeight: 'inherit', color: 'inherit' }, + '& p': { margin: '0 0 8px 0', '&:last-child': { marginBottom: 0 } }, + '& code': { fontFamily: 'monospace', backgroundColor: 'action.hover', padding: '1px 3px', borderRadius: 0.5 }, + '& pre': { fontFamily: 'monospace', backgroundColor: 'action.hover', padding: 1, overflow: 'auto', margin: '8px 0', '& code': { backgroundColor: 'transparent', padding: 0 } }, + '& a': { color: 'primary.main' }, + '& ul, & ol': { paddingLeft: 2.5, margin: '8px 0' }, + '& li': { marginBottom: 0.5 }, + '& strong': { fontWeight: 600, color: 'text.primary' }, + '& h1, & h2, & h3, & h4': { fontSize: 11.5, fontWeight: 600, color: 'text.primary', margin: '8px 0' }, + })}> + {setupDetailsContent} + )} {onDelete && connectorIdRef.current && ( diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index 2a5de336..7b27c4f0 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -58,6 +58,7 @@ import FolderOpenIcon from '@mui/icons-material/FolderOpen'; import CreateNewFolderIcon from '@mui/icons-material/CreateNewFolder'; import CloudIcon from '@mui/icons-material/Cloud'; import LanguageIcon from '@mui/icons-material/Language'; +import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'; import { useTranslation } from 'react-i18next'; import { LocalInstallUpgradePanel } from './LocalInstallUpgradePanel'; @@ -929,6 +930,63 @@ export const DataLoadMenu: React.FC = ({ // AddConnectionPanel — left sidebar lists loader types, right shows DataLoaderForm // --------------------------------------------------------------------------- +// A scrollable area that softly fades its bottom edge and shows a subtle +// down-chevron while more content sits below the fold, so long connector +// forms don't feel abruptly cut off. The affordance fades out once the user +// reaches the end. +const ScrollFadeContainer: React.FC<{ + children: React.ReactNode; + sx?: object; + /** Recompute when this changes (e.g. selected loader swaps content). */ + resetKey?: unknown; +}> = ({ children, sx, resetKey }) => { + const scrollRef = useRef(null); + const [moreBelow, setMoreBelow] = useState(false); + + const update = useCallback(() => { + const el = scrollRef.current; + if (!el) return; + setMoreBelow(el.scrollHeight - el.scrollTop - el.clientHeight > 8); + }, []); + + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + update(); + const ro = new ResizeObserver(update); + ro.observe(el); + for (const child of Array.from(el.children)) ro.observe(child); + return () => ro.disconnect(); + }, [update, resetKey]); + + return ( + + + {children} + + + `linear-gradient(to bottom, ${alpha(theme.palette.background.paper, 0)}, ${theme.palette.background.paper})`, + }} + > + + + + ); +}; + interface LoaderType { type: string; name: string; @@ -937,7 +995,7 @@ interface LoaderType { auth_mode?: string; auth_paths?: ConnectorAuthPath[]; auth_instructions?: string; - delegated_login?: { login_url: string; label?: string } | null; + delegated_login?: { login_url: string; label?: string; params?: string[] } | null; source?: 'plugin' | 'builtin'; source_path?: string | null; } @@ -1170,7 +1228,7 @@ const AddConnectionPanel: React.FC<{ ) : selectedLoader ? ( {/* Connector setup timeline */} - + - + ) : ( diff --git a/tests/backend/auth/test_kusto_oauth_gateway.py b/tests/backend/auth/test_kusto_oauth_gateway.py new file mode 100644 index 00000000..d6ba3228 --- /dev/null +++ b/tests/backend/auth/test_kusto_oauth_gateway.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +from __future__ import annotations + +import urllib.parse +from unittest.mock import Mock, patch + +import flask +import pytest + +from data_formulator.auth.gateways.kusto_oauth_gateway import kusto_oauth_bp +from data_formulator.error_handler import register_error_handlers + + +pytestmark = [pytest.mark.backend, pytest.mark.auth] + + +@pytest.fixture +def app(monkeypatch): + monkeypatch.setenv("KUSTO_OAUTH_CLIENT_ID", "kusto-client") + monkeypatch.setenv("KUSTO_OAUTH_TENANT_ID", "test-tenant") + test_app = flask.Flask(__name__) + test_app.config["TESTING"] = True + test_app.secret_key = "test-secret" + test_app.register_blueprint(kusto_oauth_bp) + register_error_handlers(test_app) + return test_app + + +@pytest.fixture +def client(app): + return app.test_client() + + +@pytest.fixture(autouse=True) +def kusto_auth_metadata(): + response = Mock(ok=True) + response.raise_for_status.return_value = None + response.json.return_value = { + "AzureAD": { + "KustoServiceResourceId": "https://kusto.kusto.windows.net", + "LoginEndpoint": "https://login.microsoftonline.com", + }, + } + with patch( + "data_formulator.auth.gateways.kusto_oauth_gateway.http.get", + return_value=response, + ): + yield + + +def _start_login(client): + return client.get( + "/api/auth/kusto/login", + query_string={ + "kusto_cluster": "https://help.kusto.windows.net", + "df_origin": "http://localhost:5173", + }, + ) + + +def test_login_uses_cluster_scope_and_pkce(client) -> None: + response = _start_login(client) + + assert response.status_code == 302 + location = urllib.parse.urlparse(response.headers["Location"]) + query = urllib.parse.parse_qs(location.query) + assert location.netloc == "login.microsoftonline.com" + assert query["client_id"] == ["kusto-client"] + assert "https://kusto.kusto.windows.net/.default" in query["scope"][0] + assert query["code_challenge_method"] == ["S256"] + assert query["code_challenge"][0] + with client.session_transaction() as current_session: + pending = current_session["kusto_oauth_state"][query["state"][0]] + assert pending["origin"] == "http://localhost:5173" + assert pending["verifier"] + + +@pytest.mark.parametrize("cluster", [ + "http://help.kusto.windows.net", + "https://help.kusto.windows.net/path", + "https://help.kusto.windows.net.evil.example", +]) +def test_login_rejects_untrusted_cluster_urls(client, cluster) -> None: + response = client.get( + "/api/auth/kusto/login", + query_string={ + "kusto_cluster": cluster, + "df_origin": "http://localhost:5173", + }, + ) + + body = response.get_json() + assert body["status"] == "error" + assert body["error"]["code"] == "INVALID_REQUEST" + + +def test_callback_exchanges_code_and_posts_token_to_opener(client) -> None: + login_response = _start_login(client) + query = urllib.parse.parse_qs( + urllib.parse.urlparse(login_response.headers["Location"]).query, + ) + state = query["state"][0] + token_response = Mock(ok=True) + token_response.json.return_value = { + "access_token": "kusto-access", + "refresh_token": "kusto-refresh", + "expires_in": 1234, + } + + with patch( + "data_formulator.auth.gateways.kusto_oauth_gateway.http.post", + return_value=token_response, + ) as post: + response = client.get( + "/api/auth/kusto/callback", + query_string={"state": state, "code": "authorization-code"}, + ) + + assert response.status_code == 200 + assert response.headers["Cache-Control"] == "no-store" + assert "kusto-access" in response.get_data(as_text=True) + token_request = post.call_args.kwargs["data"] + assert token_request["code"] == "authorization-code" + assert token_request["code_verifier"] + assert "client_secret" not in token_request + + +def test_callback_state_cannot_be_replayed(client) -> None: + login_response = _start_login(client) + query = urllib.parse.parse_qs( + urllib.parse.urlparse(login_response.headers["Location"]).query, + ) + state = query["state"][0] + + client.get( + "/api/auth/kusto/callback", + query_string={"state": state, "error": "access_denied"}, + ) + replay = client.get( + "/api/auth/kusto/callback", + query_string={"state": state, "code": "replayed-code"}, + ) + + body = replay.get_json() + assert body["status"] == "error" + assert body["error"]["code"] == "INVALID_REQUEST" \ No newline at end of file diff --git a/tests/backend/data_loader/test_databricks_connection.py b/tests/backend/data_loader/test_databricks_connection.py new file mode 100644 index 00000000..36b5b793 --- /dev/null +++ b/tests/backend/data_loader/test_databricks_connection.py @@ -0,0 +1,133 @@ +"""Tests for the Databricks Unity Catalog loader. + +Two layers: + +1. **Pure-logic unit tests** — no connection at all. They validate auth-path + declaration, param validation, the Unity Catalog hierarchy / scope pinning + and source-table parsing. + +2. **Live end-to-end test** (``@pytest.mark.live``) — exercises the real + ``databricks-sql-connector`` against an actual SQL warehouse. There is no + faithful local emulator (the SQL-warehouse protocol and Unity Catalog's + ``information_schema`` have no local stand-in), so this is the real test. + It is **skipped** unless you provide workspace credentials via env vars:: + + DATABRICKS_SERVER_HOSTNAME=adb-....azuredatabricks.net + DATABRICKS_HTTP_PATH=/sql/1.0/warehouses/xxxx + DATABRICKS_ACCESS_TOKEN=dapi.... + + The easiest zero-infra way to get these is Databricks Free Edition, which + ships the built-in ``samples`` catalog used below by default. Override the + probe target with DATABRICKS_TEST_CATALOG / DATABRICKS_TEST_SCHEMA / + DATABRICKS_TEST_TABLE if your workspace lacks ``samples``. +""" + +import os + +import pytest + +from data_formulator.data_loader.databricks_data_loader import DatabricksDataLoader +from data_formulator.data_loader.external_data_loader import ConnectorParamError + +pytestmark = [pytest.mark.backend] + + +# --------------------------------------------------------------------------- # +# Pure-logic unit tests (no connection) # +# --------------------------------------------------------------------------- # + +def _bare_loader(*, catalog: str = "", schema: str = "") -> DatabricksDataLoader: + """A loader instance with attributes set but no live connection.""" + loader = object.__new__(DatabricksDataLoader) + loader.params = {k: v for k, v in (("catalog", catalog), ("schema", schema)) if v} + loader.catalog = catalog + loader.schema = schema + return loader + + +def test_token_is_default_auth_path_without_oauth(monkeypatch) -> None: + monkeypatch.delenv("DATABRICKS_OAUTH_CLIENT_ID", raising=False) + paths = DatabricksDataLoader.auth_paths() + assert [p["id"] for p in paths] == ["token"] + assert paths[0]["default"] is True + assert DatabricksDataLoader.delegated_login_config() is None + + +def test_sign_in_path_appears_when_oauth_configured(monkeypatch) -> None: + monkeypatch.setenv("DATABRICKS_OAUTH_CLIENT_ID", "client") + paths = DatabricksDataLoader.auth_paths() + assert paths[0]["id"] == "databricks_sign_in" + assert paths[0]["kind"] == "delegated_login" + assert paths[0]["default"] is True + + +def test_token_path_requires_access_token() -> None: + params = { + "server_hostname": "adb-1.11.azuredatabricks.net", + "http_path": "/sql/1.0/warehouses/abc", + "_auth_path": "token", + } + with pytest.raises(ConnectorParamError, match="access_token"): + DatabricksDataLoader.validate_params(params) + + +def test_hierarchy_is_catalog_schema_table() -> None: + assert [h["key"] for h in DatabricksDataLoader.catalog_hierarchy()] == [ + "catalog", "schema", "table", + ] + + +def test_effective_hierarchy_pins_provided_catalog() -> None: + loader = _bare_loader(catalog="main") + assert [h["key"] for h in loader.effective_hierarchy()] == ["schema", "table"] + + +def test_resolve_source_table_variants() -> None: + loader = _bare_loader(catalog="main", schema="sales") + assert loader._resolve_source_table("c.s.t") == ("c", "s", "t") + assert loader._resolve_source_table("s2.t2") == ("main", "s2", "t2") + assert loader._resolve_source_table("t3") == ("main", "sales", "t3") + + +# --------------------------------------------------------------------------- # +# Live end-to-end test (real warehouse; skipped without credentials) # +# --------------------------------------------------------------------------- # + +_LIVE_ENV = ("DATABRICKS_SERVER_HOSTNAME", "DATABRICKS_HTTP_PATH", "DATABRICKS_ACCESS_TOKEN") +_live_ready = all(os.environ.get(k) for k in _LIVE_ENV) + + +@pytest.mark.live +@pytest.mark.skipif(not _live_ready, reason=f"set {', '.join(_LIVE_ENV)} to run the live Databricks test") +def test_live_unity_catalog_roundtrip() -> None: + catalog = os.environ.get("DATABRICKS_TEST_CATALOG", "samples") + schema = os.environ.get("DATABRICKS_TEST_SCHEMA", "nyctaxi") + table = os.environ.get("DATABRICKS_TEST_TABLE", f"{catalog}.{schema}.trips") + + loader = DatabricksDataLoader({ + "server_hostname": os.environ["DATABRICKS_SERVER_HOSTNAME"], + "http_path": os.environ["DATABRICKS_HTTP_PATH"], + "access_token": os.environ["DATABRICKS_ACCESS_TOKEN"], + "catalog": catalog, + "schema": schema, + "_auth_path": "token", + }) + + # 1) Browse: Unity Catalog listing returns 3-part names. + tables = loader.list_tables() + assert tables, "expected at least one table in the probe schema" + names = {t["name"] for t in tables} + assert table in names, f"{table} not found among {sorted(names)[:10]}" + probe = next(t for t in tables if t["name"] == table) + assert probe["path"] == table.split(".") + + # 2) Column types come back via DESCRIBE (works even where the samples + # catalog's information_schema.columns is empty). + col_types = loader.get_column_types(table) + assert col_types["columns"], "expected get_column_types to return columns" + + # 3) Fetch: real Arrow data path, bounded by LIMIT. + arrow = loader.fetch_data_as_arrow(table, import_options={"size": 5}) + assert arrow.num_rows <= 5 + assert arrow.num_columns > 0 + diff --git a/tests/backend/data_loader/test_kusto_connection.py b/tests/backend/data_loader/test_kusto_connection.py index f2c6484e..550e61cd 100644 --- a/tests/backend/data_loader/test_kusto_connection.py +++ b/tests/backend/data_loader/test_kusto_connection.py @@ -3,8 +3,12 @@ import pandas as pd import pytest -from data_formulator.data_loader.kusto_data_loader import KustoDataLoader +from data_formulator.data_loader.kusto_data_loader import ( + KustoDataLoader, + _KustoDelegatedCredential, +) from data_formulator.data_loader.external_data_loader import ConnectorParamError +from data_formulator.data_connector import DataConnector def _loader() -> KustoDataLoader: @@ -69,6 +73,74 @@ def test_ambient_path_does_not_require_service_principal_fields() -> None: KustoDataLoader.validate_params(params) +def test_microsoft_sign_in_is_default_when_oauth_is_configured(monkeypatch) -> None: + monkeypatch.setenv("KUSTO_OAUTH_CLIENT_ID", "client") + + paths = KustoDataLoader.auth_paths() + + assert paths[0]["id"] == "microsoft_sign_in" + assert paths[0]["default"] is True + assert KustoDataLoader.delegated_login_config() == { + "login_url": "/api/auth/kusto/login", + "label": "Sign in with Microsoft", + "params": ["kusto_cluster"], + } + + +def test_ambient_is_default_when_oauth_is_not_configured(monkeypatch) -> None: + monkeypatch.delenv("KUSTO_OAUTH_CLIENT_ID", raising=False) + + paths = KustoDataLoader.auth_paths() + + assert paths[0]["id"] == "ambient" + assert paths[0]["default"] is True + assert KustoDataLoader.delegated_login_config() is None + + +def test_connector_manifest_preserves_root_oauth_url(monkeypatch) -> None: + monkeypatch.setenv("KUSTO_OAUTH_CLIENT_ID", "client") + connector = DataConnector.from_loader( + KustoDataLoader, + source_id="kusto:test", + display_name="Kusto test", + ) + + config = connector.get_frontend_config() + + assert config["delegated_login"] == { + "login_url": "/api/auth/kusto/login", + "label": "Sign in with Microsoft", + "params": ["kusto_cluster"], + } + + +def test_delegated_credential_refreshes_expired_token(monkeypatch) -> None: + monkeypatch.setenv("KUSTO_OAUTH_CLIENT_ID", "client") + monkeypatch.setenv("KUSTO_OAUTH_TENANT_ID", "tenant") + response = Mock(ok=True) + response.json.return_value = { + "access_token": "new-access", + "refresh_token": "new-refresh", + "expires_in": 3600, + } + credential = _KustoDelegatedCredential( + "https://help.kusto.windows.net", + "expired-access", + "refresh", + 0, + ) + + with patch( + "data_formulator.data_loader.kusto_data_loader.http.post", + return_value=response, + ) as post: + token = credential.get_token("https://help.kusto.windows.net/.default") + + assert token.token == "new-access" + assert credential.refresh_token == "new-refresh" + assert post.call_args.kwargs["data"]["grant_type"] == "refresh_token" + + def test_legacy_complete_service_principal_infers_path() -> None: params = { "kusto_cluster": "https://example.kusto.windows.net", diff --git a/uv.lock b/uv.lock index 9d596481..30809761 100644 --- a/uv.lock +++ b/uv.lock @@ -604,7 +604,7 @@ wheels = [ [[package]] name = "data-formulator" -version = "0.8.0a2" +version = "0.8.0a3" source = { editable = "." } dependencies = [ { name = "azure-cosmos" }, @@ -614,6 +614,7 @@ dependencies = [ { name = "azure-storage-blob" }, { name = "beautifulsoup4" }, { name = "boto3" }, + { name = "databricks-sql-connector" }, { name = "db-dtypes" }, { name = "duckdb" }, { name = "flask" }, @@ -663,6 +664,7 @@ requires-dist = [ { name = "azure-storage-blob" }, { name = "beautifulsoup4" }, { name = "boto3" }, + { name = "databricks-sql-connector" }, { name = "db-dtypes" }, { name = "duckdb" }, { name = "flask" }, @@ -700,6 +702,27 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "databricks-sql-connector" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lz4" }, + { name = "oauthlib" }, + { name = "openpyxl" }, + { name = "pandas" }, + { name = "pybreaker" }, + { name = "pyjwt" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "thrift" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/e3/16d6f15f0456f6abd35406ddadd651b7579ed2f7c2ffda2722b263529427/databricks_sql_connector-4.3.0.tar.gz", hash = "sha256:047a56d31a61a04bd7eaa33a29e2c899b177f5b9089c127e9ae88c261e635cd1", size = 222828, upload-time = "2026-06-15T07:04:54.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/e5/4473f714f24d408a34c8a35f7b1e9b49d4e1c3fb2759166cee0ee23042c2/databricks_sql_connector-4.3.0-py3-none-any.whl", hash = "sha256:a48f33ab246e42950d12da0c12d7b82eae616f18e3103a4f5801ceeae8861f30", size = 251500, upload-time = "2026-06-15T07:04:52.628Z" }, +] + [[package]] name = "db-dtypes" version = "1.5.0" @@ -1783,6 +1806,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/a4/441aee36c6f6b249823d20fd91f9be9ab89d7c5a8ae542a4a4ca6d342d56/lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84", size = 3508989, upload-time = "2026-05-18T19:18:38.158Z" }, ] +[[package]] +name = "lz4" +version = "4.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/5b/6edcd23319d9e28b1bedf32768c3d1fd56eed8223960a2c47dacd2cec2af/lz4-4.4.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d6da84a26b3aa5da13a62e4b89ab36a396e9327de8cd48b436a3467077f8ccd4", size = 207391, upload-time = "2025-11-03T13:01:36.644Z" }, + { url = "https://files.pythonhosted.org/packages/34/36/5f9b772e85b3d5769367a79973b8030afad0d6b724444083bad09becd66f/lz4-4.4.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61d0ee03e6c616f4a8b69987d03d514e8896c8b1b7cc7598ad029e5c6aedfd43", size = 207146, upload-time = "2025-11-03T13:01:37.928Z" }, + { url = "https://files.pythonhosted.org/packages/04/f4/f66da5647c0d72592081a37c8775feacc3d14d2625bbdaabd6307c274565/lz4-4.4.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:33dd86cea8375d8e5dd001e41f321d0a4b1eb7985f39be1b6a4f466cd480b8a7", size = 1292623, upload-time = "2025-11-03T13:01:39.341Z" }, + { url = "https://files.pythonhosted.org/packages/85/fc/5df0f17467cdda0cad464a9197a447027879197761b55faad7ca29c29a04/lz4-4.4.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:609a69c68e7cfcfa9d894dc06be13f2e00761485b62df4e2472f1b66f7b405fb", size = 1279982, upload-time = "2025-11-03T13:01:40.816Z" }, + { url = "https://files.pythonhosted.org/packages/25/3b/b55cb577aa148ed4e383e9700c36f70b651cd434e1c07568f0a86c9d5fbb/lz4-4.4.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75419bb1a559af00250b8f1360d508444e80ed4b26d9d40ec5b09fe7875cb989", size = 1368674, upload-time = "2025-11-03T13:01:42.118Z" }, + { url = "https://files.pythonhosted.org/packages/fb/31/e97e8c74c59ea479598e5c55cbe0b1334f03ee74ca97726e872944ed42df/lz4-4.4.5-cp311-cp311-win32.whl", hash = "sha256:12233624f1bc2cebc414f9efb3113a03e89acce3ab6f72035577bc61b270d24d", size = 88168, upload-time = "2025-11-03T13:01:43.282Z" }, + { url = "https://files.pythonhosted.org/packages/18/47/715865a6c7071f417bef9b57c8644f29cb7a55b77742bd5d93a609274e7e/lz4-4.4.5-cp311-cp311-win_amd64.whl", hash = "sha256:8a842ead8ca7c0ee2f396ca5d878c4c40439a527ebad2b996b0444f0074ed004", size = 99491, upload-time = "2025-11-03T13:01:44.167Z" }, + { url = "https://files.pythonhosted.org/packages/14/e7/ac120c2ca8caec5c945e6356ada2aa5cfabd83a01e3170f264a5c42c8231/lz4-4.4.5-cp311-cp311-win_arm64.whl", hash = "sha256:83bc23ef65b6ae44f3287c38cbf82c269e2e96a26e560aa551735883388dcc4b", size = 91271, upload-time = "2025-11-03T13:01:45.016Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ac/016e4f6de37d806f7cc8f13add0a46c9a7cfc41a5ddc2bc831d7954cf1ce/lz4-4.4.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:df5aa4cead2044bab83e0ebae56e0944cc7fcc1505c7787e9e1057d6d549897e", size = 207163, upload-time = "2025-11-03T13:01:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/8d/df/0fadac6e5bd31b6f34a1a8dbd4db6a7606e70715387c27368586455b7fc9/lz4-4.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d0bf51e7745484d2092b3a51ae6eb58c3bd3ce0300cf2b2c14f76c536d5697a", size = 207150, upload-time = "2025-11-03T13:01:47.205Z" }, + { url = "https://files.pythonhosted.org/packages/b7/17/34e36cc49bb16ca73fb57fbd4c5eaa61760c6b64bce91fcb4e0f4a97f852/lz4-4.4.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7b62f94b523c251cf32aa4ab555f14d39bd1a9df385b72443fd76d7c7fb051f5", size = 1292045, upload-time = "2025-11-03T13:01:48.667Z" }, + { url = "https://files.pythonhosted.org/packages/90/1c/b1d8e3741e9fc89ed3b5f7ef5f22586c07ed6bb04e8343c2e98f0fa7ff04/lz4-4.4.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c3ea562c3af274264444819ae9b14dbbf1ab070aff214a05e97db6896c7597e", size = 1279546, upload-time = "2025-11-03T13:01:50.159Z" }, + { url = "https://files.pythonhosted.org/packages/55/d9/e3867222474f6c1b76e89f3bd914595af69f55bf2c1866e984c548afdc15/lz4-4.4.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24092635f47538b392c4eaeff14c7270d2c8e806bf4be2a6446a378591c5e69e", size = 1368249, upload-time = "2025-11-03T13:01:51.273Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e7/d667d337367686311c38b580d1ca3d5a23a6617e129f26becd4f5dc458df/lz4-4.4.5-cp312-cp312-win32.whl", hash = "sha256:214e37cfe270948ea7eb777229e211c601a3e0875541c1035ab408fbceaddf50", size = 88189, upload-time = "2025-11-03T13:01:52.605Z" }, + { url = "https://files.pythonhosted.org/packages/a5/0b/a54cd7406995ab097fceb907c7eb13a6ddd49e0b231e448f1a81a50af65c/lz4-4.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:713a777de88a73425cf08eb11f742cd2c98628e79a8673d6a52e3c5f0c116f33", size = 99497, upload-time = "2025-11-03T13:01:53.477Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7e/dc28a952e4bfa32ca16fa2eb026e7a6ce5d1411fcd5986cd08c74ec187b9/lz4-4.4.5-cp312-cp312-win_arm64.whl", hash = "sha256:a88cbb729cc333334ccfb52f070463c21560fca63afcf636a9f160a55fac3301", size = 91279, upload-time = "2025-11-03T13:01:54.419Z" }, + { url = "https://files.pythonhosted.org/packages/2f/46/08fd8ef19b782f301d56a9ccfd7dafec5fd4fc1a9f017cf22a1accb585d7/lz4-4.4.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6bb05416444fafea170b07181bc70640975ecc2a8c92b3b658c554119519716c", size = 207171, upload-time = "2025-11-03T13:01:56.595Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3f/ea3334e59de30871d773963997ecdba96c4584c5f8007fd83cfc8f1ee935/lz4-4.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b424df1076e40d4e884cfcc4c77d815368b7fb9ebcd7e634f937725cd9a8a72a", size = 207163, upload-time = "2025-11-03T13:01:57.721Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/7b3a2a0feb998969f4793c650bb16eff5b06e80d1f7bff867feb332f2af2/lz4-4.4.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:216ca0c6c90719731c64f41cfbd6f27a736d7e50a10b70fad2a9c9b262ec923d", size = 1292136, upload-time = "2025-11-03T13:02:00.375Z" }, + { url = "https://files.pythonhosted.org/packages/89/d1/f1d259352227bb1c185288dd694121ea303e43404aa77560b879c90e7073/lz4-4.4.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:533298d208b58b651662dd972f52d807d48915176e5b032fb4f8c3b6f5fe535c", size = 1279639, upload-time = "2025-11-03T13:02:01.649Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fb/ba9256c48266a09012ed1d9b0253b9aa4fe9cdff094f8febf5b26a4aa2a2/lz4-4.4.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:451039b609b9a88a934800b5fc6ee401c89ad9c175abf2f4d9f8b2e4ef1afc64", size = 1368257, upload-time = "2025-11-03T13:02:03.35Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6d/dee32a9430c8b0e01bbb4537573cabd00555827f1a0a42d4e24ca803935c/lz4-4.4.5-cp313-cp313-win32.whl", hash = "sha256:a5f197ffa6fc0e93207b0af71b302e0a2f6f29982e5de0fbda61606dd3a55832", size = 88191, upload-time = "2025-11-03T13:02:04.406Z" }, + { url = "https://files.pythonhosted.org/packages/18/e0/f06028aea741bbecb2a7e9648f4643235279a770c7ffaf70bd4860c73661/lz4-4.4.5-cp313-cp313-win_amd64.whl", hash = "sha256:da68497f78953017deb20edff0dba95641cc86e7423dfadf7c0264e1ac60dc22", size = 99502, upload-time = "2025-11-03T13:02:05.886Z" }, + { url = "https://files.pythonhosted.org/packages/61/72/5bef44afb303e56078676b9f2486f13173a3c1e7f17eaac1793538174817/lz4-4.4.5-cp313-cp313-win_arm64.whl", hash = "sha256:c1cfa663468a189dab510ab231aad030970593f997746d7a324d40104db0d0a9", size = 91285, upload-time = "2025-11-03T13:02:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/6a5c2952971af73f15ed4ebfdd69774b454bd0dc905b289082ca8664fba1/lz4-4.4.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67531da3b62f49c939e09d56492baf397175ff39926d0bd5bd2d191ac2bff95f", size = 207348, upload-time = "2025-11-03T13:02:08.117Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d7/fd62cbdbdccc35341e83aabdb3f6d5c19be2687d0a4eaf6457ddf53bba64/lz4-4.4.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a1acbbba9edbcbb982bc2cac5e7108f0f553aebac1040fbec67a011a45afa1ba", size = 207340, upload-time = "2025-11-03T13:02:09.152Z" }, + { url = "https://files.pythonhosted.org/packages/77/69/225ffadaacb4b0e0eb5fd263541edd938f16cd21fe1eae3cd6d5b6a259dc/lz4-4.4.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a482eecc0b7829c89b498fda883dbd50e98153a116de612ee7c111c8bcf82d1d", size = 1293398, upload-time = "2025-11-03T13:02:10.272Z" }, + { url = "https://files.pythonhosted.org/packages/c6/9e/2ce59ba4a21ea5dc43460cba6f34584e187328019abc0e66698f2b66c881/lz4-4.4.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e099ddfaa88f59dd8d36c8a3c66bd982b4984edf127eb18e30bb49bdba68ce67", size = 1281209, upload-time = "2025-11-03T13:02:12.091Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/4d946bd1624ec229b386a3bc8e7a85fa9a963d67d0a62043f0af0978d3da/lz4-4.4.5-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2af2897333b421360fdcce895c6f6281dc3fab018d19d341cf64d043fc8d90d", size = 1369406, upload-time = "2025-11-03T13:02:13.683Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/d429ba4720a9064722698b4b754fb93e42e625f1318b8fe834086c7c783b/lz4-4.4.5-cp313-cp313t-win32.whl", hash = "sha256:66c5de72bf4988e1b284ebdd6524c4bead2c507a2d7f172201572bac6f593901", size = 88325, upload-time = "2025-11-03T13:02:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/7ba10c9b97c06af6c8f7032ec942ff127558863df52d866019ce9d2425cf/lz4-4.4.5-cp313-cp313t-win_amd64.whl", hash = "sha256:cdd4bdcbaf35056086d910d219106f6a04e1ab0daa40ec0eeef1626c27d0fddb", size = 99643, upload-time = "2025-11-03T13:02:15.978Z" }, + { url = "https://files.pythonhosted.org/packages/77/4d/a175459fb29f909e13e57c8f475181ad8085d8d7869bd8ad99033e3ee5fa/lz4-4.4.5-cp313-cp313t-win_arm64.whl", hash = "sha256:28ccaeb7c5222454cd5f60fcd152564205bcb801bd80e125949d2dfbadc76bbd", size = 91504, upload-time = "2025-11-03T13:02:17.313Z" }, + { url = "https://files.pythonhosted.org/packages/63/9c/70bdbdb9f54053a308b200b4678afd13efd0eafb6ddcbb7f00077213c2e5/lz4-4.4.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c216b6d5275fc060c6280936bb3bb0e0be6126afb08abccde27eed23dead135f", size = 207586, upload-time = "2025-11-03T13:02:18.263Z" }, + { url = "https://files.pythonhosted.org/packages/b6/cb/bfead8f437741ce51e14b3c7d404e3a1f6b409c440bad9b8f3945d4c40a7/lz4-4.4.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c8e71b14938082ebaf78144f3b3917ac715f72d14c076f384a4c062df96f9df6", size = 207161, upload-time = "2025-11-03T13:02:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/e7/18/b192b2ce465dfbeabc4fc957ece7a1d34aded0d95a588862f1c8a86ac448/lz4-4.4.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b5e6abca8df9f9bdc5c3085f33ff32cdc86ed04c65e0355506d46a5ac19b6e9", size = 1292415, upload-time = "2025-11-03T13:02:20.829Z" }, + { url = "https://files.pythonhosted.org/packages/67/79/a4e91872ab60f5e89bfad3e996ea7dc74a30f27253faf95865771225ccba/lz4-4.4.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b84a42da86e8ad8537aabef062e7f661f4a877d1c74d65606c49d835d36d668", size = 1279920, upload-time = "2025-11-03T13:02:22.013Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/d52c7b11eaa286d49dae619c0eec4aabc0bf3cda7a7467eb77c62c4471f3/lz4-4.4.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bba042ec5a61fa77c7e380351a61cb768277801240249841defd2ff0a10742f", size = 1368661, upload-time = "2025-11-03T13:02:23.208Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/137ddeea14c2cb86864838277b2607d09f8253f152156a07f84e11768a28/lz4-4.4.5-cp314-cp314-win32.whl", hash = "sha256:bd85d118316b53ed73956435bee1997bd06cc66dd2fa74073e3b1322bd520a67", size = 90139, upload-time = "2025-11-03T13:02:24.301Z" }, + { url = "https://files.pythonhosted.org/packages/18/2c/8332080fd293f8337779a440b3a143f85e374311705d243439a3349b81ad/lz4-4.4.5-cp314-cp314-win_amd64.whl", hash = "sha256:92159782a4502858a21e0079d77cdcaade23e8a5d252ddf46b0652604300d7be", size = 101497, upload-time = "2025-11-03T13:02:25.187Z" }, + { url = "https://files.pythonhosted.org/packages/ca/28/2635a8141c9a4f4bc23f5135a92bbcf48d928d8ca094088c962df1879d64/lz4-4.4.5-cp314-cp314-win_arm64.whl", hash = "sha256:d994b87abaa7a88ceb7a37c90f547b8284ff9da694e6afcfaa8568d739faf3f7", size = 93812, upload-time = "2025-11-03T13:02:26.133Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -2154,6 +2225,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/e5/b7d20451657664b07986c2f6e3be564433f5dcaf3482d68eaecd79afaf03/numpy-2.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0", size = 12502577, upload-time = "2026-01-31T23:13:07.08Z" }, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + [[package]] name = "openai" version = "2.24.0" @@ -2545,6 +2625,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, ] +[[package]] +name = "pybreaker" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/89/fbf98e383f1ec6d117af2cd983efdb3eb7018b63834c427025764194cac2/pybreaker-1.4.1.tar.gz", hash = "sha256:8df2d245c73ba40c8242c56ffb4f12138fbadc23e296224740c2028ea9dc1178", size = 15555, upload-time = "2025-09-21T15:12:04.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/75/e64d3d40a741e2be21d69154f4e5c43a66f0c603c5ef11f49e01429a5932/pybreaker-1.4.1-py3-none-any.whl", hash = "sha256:b4dab4a05195b7f2a64a6c1a6c4ba7a96534ef56ea7210e6bcb59f28897160e0", size = 12915, upload-time = "2025-09-21T15:12:02.284Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -3378,6 +3467,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, ] +[[package]] +name = "thrift" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/c2/db648cc10dd7d15560f2eafd92a27cd280811924696e0b4a87175fb28c94/thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466", size = 62303, upload-time = "2025-05-23T20:49:33.309Z" } + [[package]] name = "tiktoken" version = "0.12.0"