From c59a0c375f41107e3bf178a1883611f4262937ae Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:47:50 +0200 Subject: [PATCH 01/12] feat(gemini): Tool Search and context compaction summaries on Open WebUI 0.12 Tool Search (Open WebUI 0.12, experimental): Open WebUI leaves tools with long definitions out of body["tools"] (utils/middleware.py:3288-3295, utils/tool_search.py:40-103), lists them in an block of the system prompt and declares search_tools, which returns their definitions; the model is expected to call a found tool by name, and Open WebUI runs it from metadata["tools"]. Gemini only calls declared functions, so the found tool could not be called (the mock answers "undeclared function", the turn ended there). _build_function_declarations now also declares every deferred tool (metadata["deferred_tools"]) that a search_tools result in the messages names, with the spec from metadata["tools"]; names are mapped like any other tool. A tool found in one round is callable in the next round and in later turns while its search result is in the history. Context compaction: Open WebUI asks the chat's model for the summary (utils/context_compaction.py:351-422, metadata task "context_compaction") and takes the answer text as the conversation summary; before a reply it saves it with the chat (context_compaction.py:122-128), and 0.12 also compacts between tool rounds (utils/middleware.py:6299-6330). The pipe answered a failed request ("Error generating content: 400 ...") or an answer without text ("[No content generated]") with that text, which then replaced the compacted messages, permanently when saved. pipe() is now a thin wrapper around the former body (_pipe) that answers the compaction task with an empty text in those cases, so Open WebUI falls back to its own summary of the compacted messages (logged as a WARNING). Version 1.20.0. +semver: minor Co-Authored-By: Claude Opus 5.5 --- pipelines/google/google_gemini.py | 151 ++++++++++++++++++++++++++++-- 1 file changed, 145 insertions(+), 6 deletions(-) diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 2e3df9e..84dad0a 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -4,7 +4,7 @@ author_url: https://github.com/owndev/ project_url: https://github.com/owndev/Open-WebUI-Functions funding_url: https://github.com/sponsors/owndev -version: 1.19.1 +version: 1.20.0 required_open_webui_version: 0.9.0 requirements: google-genai>=1.68.0, google-genai<3 license: Apache License 2.0 @@ -37,6 +37,7 @@ - Intelligent grounding with Google search integration, shown with Open WebUI's own localized search statuses (queries, searched sites) - Vertex AI Search grounding for RAG - Native tool calling through Open WebUI's tool loop (built-in, workspace, MCP, OpenAPI, terminal and direct tools, tool approval) + - Tools found by Open WebUI's Tool Search (search_tools) declared to Gemini from the next round on, so they can be called - Thought signatures carried across tool rounds and turns - Tool calls returned to API clients as OpenAI tool_calls (streaming and non-streaming) - Google Search grounding together with function calling on Gemini 3 @@ -59,7 +60,7 @@ - Image-to-video generation support for Veo models - Negative prompt and person generation controls for video - Token usage reporting for streaming and non-streaming responses - - Usable as task model for title, tag and follow-up generation + - Usable as task model for title, tag and follow-up generation and for context compaction (a failed or empty summary leaves Open WebUI's own summary in place of an error text) """ import os @@ -193,6 +194,17 @@ def _unload_stale_modules() -> None: # prefix) and channel messages. Their history exists only in the request. UNSAVED_CHAT_ID_PREFIXES = ("temporary:", "local:", "channel:") +# Open WebUI's Tool Search (0.12+, utils/tool_search.py): the tool that returns +# the definitions of the tools left out of body["tools"]. +TOOL_SEARCH_NAME = "search_tools" + +# __metadata__["task"] of Open WebUI's context compaction request, whose answer +# becomes the conversation summary (utils/context_compaction.py). +CONTEXT_COMPACTION_TASK = "context_compaction" + +# Placeholder answer when Gemini returned no text. +NO_CONTENT_TEXT = "[No content generated]" + # Seconds between two updates of the live thinking block in the chat (Open WebUI # saves the message content on every update). LIVE_THINKING_INTERVAL = 0.4 @@ -3581,7 +3593,8 @@ def _build_function_declarations( Open WebUI puts an OpenAI tool spec into body["tools"] for every tool it offers in Native mode (built-in, workspace, MCP, OpenAPI, terminal and direct tools) and runs Gemini's function calls itself; API clients send - their own tools and run the calls themselves. + their own tools and run the calls themselves. With Tool Search, the + deferred tools that search_tools found are added (_searched_tool_specs). Returns: (declarations, {gemini_name: open_webui_name}), rebuilt per request @@ -3609,6 +3622,7 @@ def _build_function_declarations( declarations: List[types.FunctionDeclaration] = [] name_map: Dict[str, str] = {} + tools = tools + self._searched_tool_specs(body.get("messages"), metadata, tools) for entry in tools: if ( not isinstance(entry, dict) @@ -3635,6 +3649,80 @@ def _build_function_declarations( declarations.append(types.FunctionDeclaration(**declaration)) return declarations, name_map + def _searched_tool_specs( + self, messages: Any, metadata: Dict[str, Any], tools: List[Any] + ) -> List[Dict[str, Any]]: + """Tool specs of the deferred tools that Open WebUI's Tool Search + returned in this conversation (Open WebUI 0.12+). + + With Tool Search on, Open WebUI leaves tools with long definitions out + of body["tools"] (__metadata__["deferred_tools"]), lists them in the + system prompt and declares search_tools, which returns their + definitions; the model is expected to call a tool it found by name. + Gemini only calls declared functions, so every deferred tool named in a + search_tools result of the messages is declared as well, with the spec + of the tool Open WebUI runs (__metadata__["tools"]). + """ + deferred = metadata.get("deferred_tools") + available = metadata.get("tools") + if not ( + isinstance(deferred, list) + and deferred + and isinstance(available, dict) + and isinstance(messages, list) + ): + return [] + declared = { + entry["function"].get("name") + for entry in tools + if isinstance(entry, dict) and isinstance(entry.get("function"), dict) + } + names: List[str] = [] + search_ids: set = set() + for message in messages: + if not isinstance(message, dict): + continue + if message.get("role") == "assistant": + # The tool messages after it answer this message's calls + search_ids = { + self._tool_call_id(call.get("id")) + for call in message.get("tool_calls") or [] + if isinstance(call, dict) + and self._tool_call_function(call).get("name") == TOOL_SEARCH_NAME + } + continue + if ( + message.get("role") != "tool" + or self._tool_call_id(message.get("tool_call_id")) not in search_ids + ): + continue + try: + result = json.loads(self._tool_result_text(message.get("content"))) + except ValueError: + continue + found = result.get("tools") if isinstance(result, dict) else None + for spec in found if isinstance(found, list) else []: + name = spec.get("name") if isinstance(spec, dict) else None + if ( + isinstance(name, str) + and name in deferred + and name not in declared + and name not in names + ): + names.append(name) + specs: List[Dict[str, Any]] = [] + for name in names: + tool = available.get(name) + spec = tool.get("spec") if isinstance(tool, dict) else None + if isinstance(spec, dict) and spec.get("name") == name: + specs.append({"type": "function", "function": spec}) + if specs: + self.log.debug( + f"Declaring {len(specs)} tool(s) found by {TOOL_SEARCH_NAME}: " + f"{[spec['function']['name'] for spec in specs]}" + ) + return specs + def _map_tool_choice( self, tool_choice: Any, name_map: Dict[str, str] ) -> Optional[types.FunctionCallingConfig]: @@ -5515,6 +5603,59 @@ async def pipe( Dict[str, Any], AsyncIterator[Union[str, Dict[str, Any]]], StreamingResponse, + ]: + """ + Entry point Open WebUI calls (see _pipe for the request path). + + Open WebUI's context compaction request gets an empty answer when + Gemini gave no summary (see _compaction_summary). + """ + result = await self._pipe( + body, __metadata__, __event_emitter__, __request__, __user__ + ) + if (__metadata__ or {}).get("task") == CONTEXT_COMPACTION_TASK and not body.get( + "stream" + ): + return self._compaction_summary(result, body.get("model", "")) + return result + + def _compaction_summary(self, result: Any, model: str) -> Any: + """The answer to Open WebUI's context compaction request. + + Open WebUI uses the answer text as the conversation summary: it replaces + the compacted messages in the request and, when compacting before a + reply, is saved with the chat for every later request. An error text, a + safety notice or the "[No content generated]" placeholder must not + become that summary, so they are answered with an empty text; Open + WebUI then uses its own summary of the compacted messages. + """ + content = None + if isinstance(result, dict): + choices = result.get("choices") + choice = choices[0] if isinstance(choices, list) and choices else None + message = choice.get("message") if isinstance(choice, dict) else None + content = message.get("content") if isinstance(message, dict) else None + if isinstance(content, str) and content.strip() and content != NO_CONTENT_TEXT: + return result + self.log.warning( + "No summary for Open WebUI's context compaction (the request failed or " + "Gemini returned no text); Open WebUI uses its own summary of the " + "compacted messages" + ) + return self._build_non_stream_result("", None, model, False) + + async def _pipe( + self, + body: Dict[str, Any], + __metadata__: dict[str, Any], + __event_emitter__: Optional[Callable], + __request__: Optional[Request] = None, + __user__: Optional[dict] = None, + ) -> Union[ + str, + Dict[str, Any], + AsyncIterator[Union[str, Dict[str, Any]]], + StreamingResponse, ]: """ Main method for sending requests to the Google Gemini endpoint. @@ -5890,9 +6031,7 @@ async def get_response(): __metadata__, ) - content = ( - full_response if full_response else "[No content generated]" - ) + content = full_response if full_response else NO_CONTENT_TEXT # Return content and usage in the shape that matches the request # (dict for stream=false, content + usage chunks for stream=true) From 9c8389505d07e26fda6c64e4864ba62414ee5ae6 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:49:42 +0200 Subject: [PATCH 02/12] feat(gemini): shared chats with replies follow Open WebUI's access checks Open WebUI 0.12 lets a chat or folder be shared with "Allow replies", so that its participants write in the owner's chat. The image history and the file reads of the pipeline accepted only the owner or an admin, so a participant's edit request lacked the chat's generated images. With Open WebUI's own checks (detected by their shape: an async Chats.get_accessible_chat_by_id with a permission keyword and an async has_access_to_file), the pipeline now - reads a saved chat's history when Open WebUI lets the user continue the chat: get_accessible_chat_by_id(chat_id, user, permission="write"), the check chat_completion makes before it calls the pipe (an admin only reads other users' chats in 0.12, so no longer continues them); - reads a file of another user only for an admin or when has_access_to_file(file_id, "read", user) allows it, Open WebUI's own rule for file downloads; - links its generated images to their chat message (chat_file), as Open WebUI does for its own image generation and uploads, so participants may read them in the web UI and in their image history. Errors of the checks refuse. Older Open WebUI versions keep the owner / admin rule unchanged. +semver: minor Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/google-gemini-integration.md | 7 +- pipelines/google/google_gemini.py | 178 +++++++++++++++++++++++++++--- 3 files changed, 168 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index b4428ee..ba6609f 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ The functions include a built-in encryption mechanism for sensitive information: - Provides configurable error handling and timeouts: temporary API errors are retried (`GOOGLE_RETRY_COUNT`) for streaming and non-streaming requests, and every started status (image, video) is closed when a request fails or is stopped. - **Advanced image processing**: Optimized image handling with configurable compression, resizing, and quality settings. - **Configurable parameters**: Environment variables for image optimization (quality, max dimensions, format conversion). -- **Multi-image history**: Image edits across turns: the generated and uploaded images of a saved chat are sent with the edit request (temporary chats send only your own uploads). Configurable image limit that keeps the current and the newest images, hash-based deduplication, and automatic `[Image N]` labels so the model can reference earlier images. +- **Multi-image history**: Image edits across turns: the generated and uploaded images of a saved chat are sent with the edit request (temporary chats send only your own uploads), also for everyone who replies in a chat shared with "Allow replies" (Open WebUI 0.12; who may read a chat or file is decided by Open WebUI's own access checks). Configurable image limit that keeps the current and the newest images, hash-based deduplication, and automatic `[Image N]` labels so the model can reference earlier images. - **Image generation (Gemini 3)**: Configurable aspect ratio (for example `16:9` or `1:1`) and resolution (`1K`, `2K`, or `4K`) for Gemini 3 image models, with per-user valve overrides. - **Nano Banana image models**: `gemini-nano-banana-2.1`, `gemini-3.1-flash-image` and `gemini-3.1-flash-lite-image` are detected as image models; each generated image is uploaded once (interim thought images are skipped unless no final image arrives). API clients get generated images and videos as links in the answer. Google Search grounding is left out for the image models without Search support (`gemini-2.5-flash-image`, `gemini-3.1-flash-lite-image`). Newer image models outside Gemini 3 / Nano Banana naming can be added via `GOOGLE_IMAGE_GENERATION_MODELS` without a code change. - **Video generation (Veo)**: Generate videos with Google Veo models (3.1, 3, 2). Configurable aspect ratio, resolution, duration, negative prompt, and person generation controls. Supports text-to-video and image-to-video for all supported Veo models. Videos are automatically uploaded and embedded with playback controls. diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index 3c828ee..8683e8c 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -430,12 +430,17 @@ Where the images of earlier turns come from: - **Saved chats** (the normal web UI chat): from the chat as Open WebUI stored it. That covers the generated images, which are attached to the answers as files (Open WebUI does not pass the files of earlier answers to a pipe), images you uploaded, images linked in earlier answers (pipeline versions before 1.15.2) and, when you regenerate an answer with guidance, the images of the user message whose answer you regenerate. Background tasks such as titles and follow-up suggestions get no images. - **API clients** (no saved chat): from the request's messages, as `image_url` parts (data URLs or Open WebUI file URLs) or markdown image links in the content, such as the `![Generated Image](/api/v1/files//content)` link the pipeline returns. - **Temporary chats**: Open WebUI does not save them, and the web UI sends only the images of your own messages. A generated image of an earlier turn is therefore not sent. To edit it, download it and attach it to your edit message. +- **Shared chats with "Allow replies"** (Open WebUI 0.12 and later): the people a chat or its folder is shared with in this mode write in the owner's chat, and their edit requests get the images of the whole chat like the owner's: the owner's uploads, the generated images of every answer and the images of the other participants. The pipeline links the images it generates to their chat, as Open WebUI does for its own image generation and for uploads, so the participants also see them. Images generated before pipeline 1.20.0 are not linked: Open WebUI shows them only to their owner and admins, and the pipeline does not send them for the other participants. A "Clone only" share or a read-only folder share lets people read the chat but not write in it: Open WebUI refuses their messages before the pipeline runs. Which images are sent: - `GOOGLE_IMAGE_HISTORY_MAX_REFERENCES` (default 5) limits the images per request. The images of the current message are always kept, the rest of the limit goes to the newest images of earlier turns, and older ones are dropped first. So the image you want to edit stays in the request in a long chat. Earlier images are read newest first and only as many as fit. - Identical images are sent once (`GOOGLE_IMAGE_DEDUP_HISTORY`): an image you attach again (for example a downloaded generated image) is not repeated from the history, and an image of several turns counts at its newest place. Images of earlier turns come before the current message's images unless `GOOGLE_IMAGE_HISTORY_FIRST=false`. -- The pipeline reads only the requesting user's own files and chats (an admin may read every file and chat), also when a message or an API request names the file id of another user. Open WebUI 0.9.6 and later check the images they put into the request themselves in the same way (and also allow files shared with the user). Open WebUI 0.9.0 to 0.9.5 put the image of any file id named in a user message into the request before the pipeline runs, so update Open WebUI if users must not reach each other's files. +- Access follows Open WebUI: the pipeline never reads a chat or file that Open WebUI would not let the requesting user use, also when a message or an API request names the file id of another user. + - Open WebUI 0.12 and later (detected by its access checks, not by the version): the history of a saved chat is read when Open WebUI lets the user continue the chat (`Chats.get_accessible_chat_by_id(..., permission="write")`, which Open WebUI itself checks before it calls the pipeline): the owner and the participants of a chat or folder shared with "Allow replies", users or groups. An admin is no exception: Open WebUI 0.12 gives admins read access to other users' chats, not write access. A file is read when it is the user's own, the user is an admin, or Open WebUI's `has_access_to_file` lets the user read it (for example through a shared chat or folder it is attached to, a knowledge base, a channel or a shared model), the same rule as Open WebUI's own file download. + - Older Open WebUI versions: only the requesting user's own files and chats (an admin may read every file and chat). + - A file the user may not read is skipped and logged without its id. An error in an access check counts as a refusal: the chat's images then come from the request, as for API clients. + - Open WebUI 0.9.6 and later check the images they put into the request themselves in the same way (and also allow files shared with the user). Open WebUI 0.9.0 to 0.9.5 put the image of any file id named in a user message into the request before the pipeline runs, so update Open WebUI if users must not reach each other's files. - Earlier images are sent as parts of your message, which is how Gemini takes input images for editing. Earlier answers are not replayed as model turns, so no thought signatures are needed. ## Video Generation Configuration diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 2e3df9e..fcb79c9 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -4,7 +4,7 @@ author_url: https://github.com/owndev/ project_url: https://github.com/owndev/Open-WebUI-Functions funding_url: https://github.com/sponsors/owndev -version: 1.19.1 +version: 1.20.0 required_open_webui_version: 0.9.0 requirements: google-genai>=1.68.0, google-genai<3 license: Apache License 2.0 @@ -17,7 +17,8 @@ - Retries of temporary API errors for streaming and non-streaming requests - Advanced multimodal input support (text and images) - Unified image generation and editing with Gemini 2.5 Flash Image Preview - - Image editing across turns: earlier generated and uploaded images of a saved chat are sent with the edit request (the newest kept within the image limit, each image once; the pipeline reads only files of the requesting user) + - Image editing across turns: earlier generated and uploaded images of a saved chat are sent with the edit request (the newest kept within the image limit, each image once; only files the requesting user may read) + - Shared chats with "Allow replies" (Open WebUI 0.12+): participants edit the images of the chat too; who may continue a chat and read its files follows Open WebUI's own access checks, generated images are linked to the chat like Open WebUI's own - Nano Banana image models (gemini-3.1-flash-image, gemini-3.1-flash-lite-image, gemini-nano-banana-2.1) - Extra image generation model IDs configurable without a code change - Interim thought images skipped, so each generated image is uploaded once (the last one is kept if no final image arrives) @@ -72,6 +73,7 @@ import hashlib import importlib import importlib.metadata +import inspect import json import logging import io @@ -107,6 +109,44 @@ except ImportError: # without it, image history comes from the request only get_message_list = None +try: # Open WebUI's own file access check (used as _shared_chat_access says) + from open_webui.utils.access_control import files as owui_file_access +except Exception: # without it, files are read for their owner and admins only + owui_file_access = None + + +def _shared_chat_access() -> bool: + """Whether Open WebUI's own checks decide who may continue a chat and read + the files of other users. + + Open WebUI 0.12 lets a chat or folder be shared with "Allow replies": its + participants write in the owner's chat. Its checks are then + Chats.get_accessible_chat_by_id(id, user, permission="write") for continuing + a chat (what chat_completion asks before it calls a pipe) and, for a file + that is not the user's own, has_access_to_file(file_id, "read", user). + Detected by their shape, not by a version: get_accessible_chat_by_id with a + ``permission`` keyword (0.12+) and an async has_access_to_file (0.11 has + one too, without the shared chats and folders of 0.12, so it is used only + together with the chat check). Without them the pipeline keeps its own + rule: the owner or an admin, as Open WebUI up to 0.11. + """ + check_chat = getattr(Chats, "get_accessible_chat_by_id", None) + check_file = getattr(owui_file_access, "has_access_to_file", None) + if check_chat is None or check_file is None: + return False + try: + parameters = inspect.signature(check_chat).parameters + except (TypeError, ValueError): + return False + return ( + "permission" in parameters + and inspect.iscoroutinefunction(check_chat) + and inspect.iscoroutinefunction(check_file) + ) + + +OWUI_SHARED_CHAT_ACCESS = _shared_chat_access() + def _unload_stale_modules() -> None: """ @@ -662,10 +702,10 @@ async def _load_saved_chat_chain( Returns None when there is no saved chat (API clients, temporary and channel chats), for background tasks (titles, follow-ups and the like - get Open WebUI's task prompt, not the chat's images), when the chat - belongs to another user (admins may use any chat, like in Open WebUI), - on Open WebUI versions without get_message_list and on errors: the - image history then comes from the request's messages. + get Open WebUI's task prompt, not the chat's images), when the user + may not continue the chat (see _may_continue_chat), on Open WebUI + versions without get_message_list and on errors (also of the access + check): the image history then comes from the request's messages. """ metadata = __metadata__ or {} user = __user__ or {} @@ -681,9 +721,7 @@ async def _load_saved_chat_chain( ): return None try: - if user.get("role") != "admin" and not await Chats.is_chat_owner( - chat_id, user["id"] - ): + if not await self._may_continue_chat(chat_id, user): return None messages_map = await Chats.get_messages_map_by_chat_id(chat_id) chain = get_message_list(messages_map or {}, user_message_id) @@ -692,6 +730,62 @@ async def _load_saved_chat_chain( return None return [m for m in chain if isinstance(m, dict)] or None + @staticmethod + async def _may_continue_chat(chat_id: str, user: Dict[str, Any]) -> bool: + """Whether ``user`` may continue the saved chat, so that its history + (files of other users included) may be read for the image history. + + With Open WebUI's own checks (_shared_chat_access, Open WebUI 0.12+) + exactly what Open WebUI asks before it calls the pipe: + Chats.get_accessible_chat_by_id(chat_id, user, permission="write"), + which allows the owner and the participants of a chat or folder shared + with "Allow replies" (users or groups), but an admin only like any + other user (read access to other users' chats does not allow + continuing them). Otherwise the owner or an admin, as Open WebUI up to + 0.11. Errors propagate; the caller refuses then. + """ + if OWUI_SHARED_CHAT_ACCESS: + account = await Users.get_user_by_id(user["id"]) + if account is None: + return False + chat = await Chats.get_accessible_chat_by_id( + chat_id, account, permission="write" + ) + return bool(chat) + if user.get("role") == "admin": + return True + return bool(await Chats.is_chat_owner(chat_id, user["id"])) + + async def _may_read_file(self, file_obj: Any, user: Dict[str, Any]) -> bool: + """Whether ``user`` may read an Open WebUI file (``file_obj``, whose id a + message may name freely, e.g. in a markdown link of an API client). + + Like Open WebUI's own GET /api/v1/files/{id}/content: the owner, an + admin and, with Open WebUI's own checks (_shared_chat_access, Open WebUI + 0.12+), whoever has_access_to_file(id, "read", user) lets read the file + (e.g. the participants of a shared chat it is attached to, the members + of a knowledge base or channel). Errors refuse, and the file id stays + out of the log. + """ + user_id = user.get("id") + if (user_id and file_obj.user_id == user_id) or user.get("role") == "admin": + return True + if not user_id or not OWUI_SHARED_CHAT_ACCESS: + return False + try: + account = await Users.get_user_by_id(user_id) + if account is None: + return False + allowed = await owui_file_access.has_access_to_file( + file_obj.id, "read", account + ) + except Exception as e: + self.log.warning( + f"File access check failed for user {user_id}: {type(e).__name__}" + ) + return False + return allowed is True + @staticmethod def _saved_image_file_url(file: Any) -> Optional[str]: """Where to read an image file of a saved message from. @@ -1820,6 +1914,51 @@ async def _emit_generated_image_files( self.log.warning(f"Failed to emit generated image files: {emit_error}") return False + async def _link_chat_files( + self, + files: List[Dict[str, Any]], + __metadata__: Optional[Dict[str, Any]], + ) -> None: + """Link uploaded generated files to their chat message (chat_file rows). + + Open WebUI does the same for the images of its own image generation + (routers/images.py upload_image) and for uploads, and this pipeline for + Veo videos. Open WebUI's file access follows these links: the + participants of a chat shared with "Allow replies" (and the viewers of + a shared chat) may then read the images, in the web UI and in the image + history of their turns. Open WebUI links a file only when the user may + read it. Saved chats only. + """ + metadata = __metadata__ or {} + chat_id = str(metadata.get("chat_id") or "") + message_id = metadata.get("message_id") + user_id = metadata.get("user_id") + if ( + not chat_id + or not message_id + or not user_id + or chat_id.startswith(UNSAVED_CHAT_ID_PREFIXES) + ): + return + file_ids = [] + for entry in files: + match = re.search(r"/files/([A-Za-z0-9_-]+)", str(entry.get("url") or "")) + if match and match.group(1) not in file_ids: + file_ids.append(match.group(1)) + if not file_ids: + return + try: + await Chats.insert_chat_files( + chat_id=chat_id, + message_id=message_id, + file_ids=file_ids, + user_id=user_id, + ) + except Exception as e: + self.log.warning( + f"Failed to link generated image files to the chat message: {e}" + ) + async def _emit_generated_video_files( self, video_files: List[Dict[str, Any]], @@ -1933,6 +2072,8 @@ async def _append_generated_images( files_emitted = await self._emit_generated_image_files( generated_image_files, __event_emitter__ ) + if files_emitted: + await self._link_chat_files(generated_image_files, __metadata__) if generated_image_files and not files_emitted: generated_images.extend( @@ -3144,9 +3285,11 @@ async def _fetch_file_as_base64( """ Fetch a file from Open WebUI's file system and convert to base64. - Only files of the requesting user are read (an admin may read every - file), like Open WebUI's own image resolver: a message can name any - file id, for example in a markdown link sent by an API client. + Only files the requesting user may read are read (see _may_read_file: + the user's own files, every file for an admin, and on Open WebUI 0.12+ + the files Open WebUI lets the user read, e.g. those of a chat shared + with the user): a message can name any file id, for example in a + markdown link sent by an API client. Args: file_url: File URL from Open WebUI @@ -3166,15 +3309,16 @@ async def _fetch_file_as_base64( file_obj = await Files.get_file_by_id(fid) user = __user__ or {} - if ( - file_obj - and file_obj.user_id != user.get("id") - and user.get("role") != "admin" - ): + if file_obj and not await self._may_read_file(file_obj, user): # the file id of another user stays out of the log self.log.warning( f"Not reading a file for user {user.get('id') or '?'}: it " "does not belong to the requesting user" + + ( + " and Open WebUI grants no read access to it" + if OWUI_SHARED_CHAT_ACCESS + else "" + ) ) return None if file_obj and file_obj.path: From e0d2adaa4bb5cb166ba91629d90b439065f7b6a6 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:49:43 +0200 Subject: [PATCH 03/12] test(e2e): gemini shared group for chats shared with replies suites/_gemini_shared.py drives real shares on Open WebUI 0.12: a participant (user grant, Allow replies), the owner after the participant and a group member edit with every image of the chat; a Clone only viewer (also with a read-only folder share) and an outsider are refused by Open WebUI. A test-only global filter hands the pipe the chat without Open WebUI's check, so the pipe's own check is tested for the admin, the read-only viewer and the outsider, and injects errors into both access helpers. Markdown links: the owner's upload of the shared chat is read, foreign files are not (logged without their ids); Veo image-to-video. On Open WebUI 0.11 the group reports that the feature is missing and checks that the owner / admin rule stays. With google_gemini.py 1.19.1 on v0.12.0-slim 10 of the 13 checks fail. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/suites/_gemini_shared.py | 649 +++++++++++++++++++++++++++++ tests/e2e/suites/gemini.py | 15 + 2 files changed, 664 insertions(+) create mode 100644 tests/e2e/suites/_gemini_shared.py diff --git a/tests/e2e/suites/_gemini_shared.py b/tests/e2e/suites/_gemini_shared.py new file mode 100644 index 0000000..56e11e7 --- /dev/null +++ b/tests/e2e/suites/_gemini_shared.py @@ -0,0 +1,649 @@ +""" +Group ``shared`` of the gemini suite: image editing in chats that Open WebUI +0.12 shares with "Allow replies" (google_gemini.py 1.20.0). Run by +suites/gemini.py; the leading underscore keeps this module out of the suite +discovery. + +Open WebUI 0.12 shares a chat with users or groups (``POST +/api/v1/chats//share`` with ``share_mode: "continue"``, then ``POST +/api/v1/chats/shared//access/update`` with the grants) or a folder (``POST +/api/v1/folders//access/update``). The participants of a "continue" share +write in the owner's chat: chat_completion (main.py) asks +``Chats.get_accessible_chat_by_id(chat_id, user, permission="write")`` before +it calls the pipe. A share without "continue" (Clone only) and a read-only +folder share give read access only. The pipe loads the image history of a chat +with the same check and reads a file of another user only for an admin or when +``has_access_to_file(file_id, "read", user)`` lets the user read it (Open +WebUI's own rule, routers/files.py). It links its generated images to the chat +(chat_file) like Open WebUI's own image generation, so the participants may read +them; Open WebUI links uploads itself. + +Users (all non-admin): owner A, participant B (user grant on chat X), group +member G (group grant on chat X), viewer D (chat Y shared Clone only with D, +and Y in a folder shared read-only with D) and outsider C. Chat X: A's turn +with the upload "upload1" (generated image "final"), then B, A and G write in +it. Chat Y: two turns of A (upload "upload2", generated "gen1"). + +The test-only global filter REDIRECT_FILTER hands the pipe another chat in +``__metadata__`` (``e2e-chat=/`` in the prompt of an +API request), as if Open WebUI had called it for that chat without its own +check: so the pipe's own check is tested also for the users Open WebUI turns +away before the pipe (the admin, the read-only viewer, the outsider). Its +fault markers make the pipe's next call of an access helper raise. + +On Open WebUI before 0.12 (no "Allow replies") the group records +``shared.unsupported`` (PASS) and ``shared.legacy``: the pipe keeps its owner / +admin rule there (also checked by the imgedit group), although Open WebUI 0.11 +lets a user a chat is shared with read its files. +""" + +from harness import BrowserSession, Suite, short +from harness.known import version_tuple + +from .gemini import ( + FOREIGN_PNG, + IMAGE_GA, + LEGACY, + SAVED_PNG, + UPLOAD_PNG_1, + UPLOAD_PNG_2, + VEO, + _edit_report, + _edit_request, + _gen, + _grant_read, + _upload_png, + _video_image, +) + +OWNER = "e2e-shared-owner@example.com" +PARTICIPANT = "e2e-shared-participant@example.com" +MEMBER = "e2e-shared-member@example.com" +VIEWER = "e2e-shared-viewer@example.com" +OUTSIDER = "e2e-shared-outsider@example.com" +GROUP_NAME = "E2E Shared Chats" +FOLDER_NAME = "E2E Shared Folder" +EDIT = "Change the name to Bob" +# what the edit request sends with the history of chat X's first turn +WITH_HISTORY = [EDIT, "[Image 1]", "upload1", "[Image 2]", "final"] +FILE_REFUSED = "does not belong to the requesting user" + +REDIRECT_FILTER_ID = "e2e_chat_redirect" +REDIRECT_FILTER = '''""" +title: E2E Chat Redirect +author: owndev +version: 0.1.0 +license: Apache License 2.0 +description: Test-only filter for tests/e2e (gemini shared). In an API request (no message id), "e2e-chat=/" in the last user message puts that chat id and user message id into the pipe's __metadata__. "e2e-chat-fault=" / "e2e-file-fault=" make the next call of Chats.get_accessible_chat_by_id / has_access_to_file for that id from the Gemini function raise RuntimeError, once (within 60 s). The markers are removed from the message. +""" + +import functools +import re +import sys +import time +from typing import Optional + +REDIRECT = re.compile(r"\\s*e2e-chat=([A-Za-z0-9-]+)/([A-Za-z0-9-]+)") +FAULT = re.compile(r"\\s*e2e-(chat|file)-fault=([A-Za-z0-9-]+)") + + +def _arm(owner, name: str, key: str, message: str) -> None: + """Wrap owner. once (an async wrapper with the same signature, so the + pipe's feature detection still sees the helper; it stays in place, inert + unless armed) and arm it for ``key`` (its first argument).""" + wrapper = getattr(owner, name) + if not hasattr(wrapper, "e2e_armed"): + original = wrapper + + @functools.wraps(original) + async def wrapper(*args, **kwargs): + caller = sys._getframe(1).f_globals.get("__name__", "") + armed = wrapper.e2e_armed + if caller.startswith("function_gemini") and args and args[0] in armed: + if time.time() < armed.pop(args[0]): + raise RuntimeError(message) + return await original(*args, **kwargs) + + wrapper.e2e_armed = {} + setattr(owner, name, wrapper) + wrapper.e2e_armed[key] = time.time() + 60 + + +class Filter: + async def inlet(self, body: dict, __metadata__: Optional[dict] = None) -> dict: + messages = body.get("messages") or [] + text = messages[-1].get("content") if messages else None + if __metadata__ is None or not isinstance(text, str): + return body + for kind, key in FAULT.findall(text): + if kind == "chat": + from open_webui.models.chats import Chats + + _arm(Chats, "get_accessible_chat_by_id", key, "e2e chat access fault") + else: + from open_webui.utils.access_control import files + + _arm(files, "has_access_to_file", key, "e2e file access fault") + text = FAULT.sub("", text) + match = REDIRECT.search(text) + if match and not __metadata__.get("message_id"): + __metadata__["chat_id"], __metadata__["user_message_id"] = match.groups() + text = REDIRECT.sub("", text) + messages[-1]["content"] = text.strip() + return body +''' + + +def _grant(principal_type: str, principal_id: str) -> dict: + return { + "principal_type": principal_type, + "principal_id": principal_id, + "permission": "read", + } + + +async def _share(owner, chat_id: str, grants: list, replies: bool) -> list: + """Share a chat like the web UI's share dialog: the share link (with + share_mode "continue" for "Allow replies"), then who may open it. + Returns the HTTP statuses.""" + mode = {"share_mode": "continue"} if replies else {} + s1, _ = await owner.api("POST", f"/api/v1/chats/{chat_id}/share", mode) + s2, _ = await owner.api( + "POST", + f"/api/v1/chats/shared/{chat_id}/access/update", + {"access_grants": grants, **mode}, + ) + return [s1, s2] + + +async def _group(t: Suite, member_id: str) -> tuple: + """(group id, HTTP status of adding the member); the group is reused.""" + _, groups = await t.owui.api("GET", "/api/v1/groups/") + found = [ + g for g in groups or [] if isinstance(g, dict) and g.get("name") == GROUP_NAME + ] + if found: + group_id = found[0]["id"] + else: + _, created = await t.owui.api( + "POST", + "/api/v1/groups/create", + {"name": GROUP_NAME, "description": "e2e shared chats"}, + ) + group_id = created.get("id", "") if isinstance(created, dict) else "" + status, _ = await t.owui.api( + "POST", f"/api/v1/groups/id/{group_id}/users/add", {"user_ids": [member_id]} + ) + return group_id, status + + +async def _read_only_folder(owner, chat_id: str, viewer_id: str) -> list: + """Move the chat into a folder of the owner shared read-only (no "continue") + with the viewer. Returns the HTTP statuses.""" + _, folders = await owner.api("GET", "/api/v1/folders/") + found = [ + f for f in folders or [] if isinstance(f, dict) and f.get("name") == FOLDER_NAME + ] + if found: + s1, folder_id = 200, found[0]["id"] + else: + s1, created = await owner.api("POST", "/api/v1/folders/", {"name": FOLDER_NAME}) + folder_id = created.get("id", "") if isinstance(created, dict) else "" + s2, _ = await owner.api( + "POST", f"/api/v1/chats/{chat_id}/folder", {"folder_id": folder_id} + ) + s3, _ = await owner.api( + "POST", + f"/api/v1/folders/{folder_id}/access/update", + {"access_grants": [_grant("user", viewer_id)]}, + ) + return [s1, s2, s3] + + +async def _file_status(client, file_id: str) -> int: + return (await client.request("GET", f"/api/v1/files/{file_id}/content")).status_code + + +def _target(chat) -> str: + """``e2e-chat=/`` of a browser turn.""" + return f"e2e-chat={chat.chat_id}/{(chat.message or {}).get('parentId')}" + + +async def _api_edit(client, mock, text: str) -> tuple: + """An API request to the image model: (result, what it sent upstream).""" + await mock.reset() + result = await client.chat(IMAGE_GA, text, stream=False) + return result, await _edit_request(mock) + + +def _brief(chat) -> str: + if chat is None: + return "-" + return f"HTTP {chat.http_status} done={chat.done}" + + +async def legacy(t: Suite, mock, version: str) -> None: + """Open WebUI before 0.12: no "Allow replies", and has_access_to_file + (0.11) knows only share links with grants. The pipe keeps its owner / admin + rule there: a user the chat is shared with reads the chat's files in Open + WebUI (the generated image too, linked by the pipe), but a markdown link of + that user to the owner's upload is not read.""" + t.check( + "shared.unsupported", + f"shared chats with replies: feature not in Open WebUI {version}", + True, + f"owui={version}", + ) + error, users, grants, shares, visible = "", {}, [], [], {} + c1 = result = None + req = {"contents": 0, "sent": [], "images": []} + denied = 0 + try: + for key, name, email in ( + ("owner", "E2E Shared Owner", OWNER), + ("participant", "E2E Shared Participant", PARTICIPANT), + ): + users[key] = await t.owui.create_user(name, email) + owner, part = users["owner"], users["participant"] + grants = [await _grant_read(t, IMAGE_GA, "Gemini 3.1 Flash Image")] + await t.owui.models(refresh=True) + upload = await _upload_png(owner, UPLOAD_PNG_1, "upload1.png") + async with BrowserSession(owner) as b: + await mock.reset() + c1 = await b.chat( + IMAGE_GA, + "Draw a certificate like this", + params=LEGACY, + user_files=[upload], + ) + shares = await _share( + owner, c1.chat_id, [_grant("user", part.user["id"])], replies=False + ) + generated = [ + (f.get("url") or "").split("/files/")[-1].split("/")[0] + for f in c1.files + if f.get("type") == "image" + ] + visible = { + "upload": await _file_status(part, upload["id"]), + "generated": [await _file_status(part, fid) for fid in generated], + } + mark = t.mark() + result, req = await _api_edit( + part, mock, f"Edit this ![a](/api/v1/files/{upload['id']}/content)" + ) + await t.log.settle(0.5) + denied = len(t.log.lines(mark, FILE_REFUSED)) + except Exception as exc: # noqa: BLE001 - reported in the check + error = f"setup failed: {exc!r} " + finally: + for client in users.values(): + await client.close() + await t.owui.delete_model(IMAGE_GA) + t.check( + "shared.legacy", + f"Open WebUI {version}: a user the chat is shared with reads the owner's " + "upload and generated image in Open WebUI, but the pipe keeps its owner / " + "admin rule: the user's markdown link to the upload is not read", + not error + and grants == [200] + and shares == [200, 200] + and visible.get("upload") == 200 + and visible.get("generated") == [200] + and getattr(result, "status", None) == 200 + and req["images"] == [] + and denied > 0, + f"{error}grants={grants} shares={shares} turn1: {_brief(c1)} " + f"participant: upload HTTP {visible.get('upload')} generated HTTP " + f"{visible.get('generated')} link: HTTP {getattr(result, 'status', None)} " + f"{_edit_report(req)} denied_logged={denied}", + ) + + +async def shared(t: Suite, mock) -> None: + version = await t.owui.version() + if version_tuple(version) < (0, 12): + await legacy(t, mock, version) + return + empty = {"contents": 0, "sent": [], "images": []} + error = "" + users: dict = {} + grants: list = [] + shares: list = [] + group_status = folder = None + visible: dict = {} + cx1 = cb = ca = cg = cy1 = cy2 = cd = cc = None + rb = ra = rg = empty + viewer_calls = outsider_calls = -1 + redirected: dict = {} + links = video = chat_fault = file_fault = None + links_req = file_fault_req = chat_fault_req = empty + video_image = "" + denied = chat_fault_logged = file_fault_logged = 0 + ids_logged: list = [] + group_id = "" + try: + for key, name, email in ( + ("owner", "E2E Shared Owner", OWNER), + ("participant", "E2E Shared Participant", PARTICIPANT), + ("member", "E2E Shared Member", MEMBER), + ("viewer", "E2E Shared Viewer", VIEWER), + ("outsider", "E2E Shared Outsider", OUTSIDER), + ): + users[key] = await t.owui.create_user(name, email) + owner, part = users["owner"], users["participant"] + grants = [ + await _grant_read(t, IMAGE_GA, "Gemini 3.1 Flash Image"), + await _grant_read(t, VEO, "Veo 3.1 Generate Preview"), + ] + await t.owui.models(refresh=True) + status, _ = await t.owui.install_function( + REDIRECT_FILTER_ID, "E2E Chat Redirect", REDIRECT_FILTER + ) + grants += [ + status, + await t.owui.set_active(REDIRECT_FILTER_ID, True), + await t.owui.set_global(REDIRECT_FILTER_ID, True), + ] + group_id, group_status = await _group(t, users["member"].user["id"]) + upload = await _upload_png(owner, UPLOAD_PNG_1, "upload1.png") + upload_y = await _upload_png(owner, UPLOAD_PNG_2, "upload2.png") + private = await _upload_png(owner, SAVED_PNG, "private.png") + admin_file = await _upload_png(t.owui, FOREIGN_PNG, "admin.png") + + # chat X: the owner's first turn, shared with "Allow replies" + async with BrowserSession(owner) as b: + await mock.reset() + cx1 = await b.chat( + IMAGE_GA, + "Draw a certificate like this", + params=LEGACY, + user_files=[upload], + ) + cy1 = await b.chat( + IMAGE_GA, + "Draw a badge like this final-image-1", + params=LEGACY, + user_files=[upload_y], + ) + cy2 = await b.chat( + IMAGE_GA, + "Make it round", + params=LEGACY, + chat_id=cy1.chat_id, + parent_id=cy1.message_id, + ) + shares = await _share( + owner, + cx1.chat_id, + [_grant("user", part.user["id"]), _grant("group", group_id)], + replies=True, + ) + shares += await _share( + owner, cy1.chat_id, [_grant("user", users["viewer"].user["id"])], False + ) + folder = await _read_only_folder(owner, cy1.chat_id, users["viewer"].user["id"]) + generated = [ + (f.get("url") or "").split("/files/")[-1].split("/")[0] + for f in cx1.files + if f.get("type") == "image" + ] + visible = { + "upload": await _file_status(part, upload["id"]), + "generated": [await _file_status(part, fid) for fid in generated], + } + + # B, A and G continue chat X in the browser + async with BrowserSession(part) as b: + await mock.reset() + cb = await b.chat( + IMAGE_GA, + f"{EDIT} final-image-2", + params=LEGACY, + chat_id=cx1.chat_id, + parent_id=cx1.message_id, + ) + rb = await _edit_request(mock) + async with BrowserSession(owner) as b: + await mock.reset() + ca = await b.chat( + IMAGE_GA, + "Make it blue final-image-3", + params=LEGACY, + chat_id=cx1.chat_id, + parent_id=cb.message_id, + ) + ra = await _edit_request(mock) + async with BrowserSession(users["member"]) as b: + await mock.reset() + cg = await b.chat( + IMAGE_GA, + "Add a frame", + params=LEGACY, + chat_id=cx1.chat_id, + parent_id=ca.message_id, + ) + rg = await _edit_request(mock) + # Open WebUI turns away the Clone-only viewer and the outsider + async with BrowserSession(users["viewer"]) as b: + await mock.reset() + cd = await b.chat( + IMAGE_GA, + EDIT, + params=LEGACY, + chat_id=cy1.chat_id, + parent_id=cy2.message_id, + ) + viewer_calls = len(await mock.requests(_gen)) + async with BrowserSession(users["outsider"]) as b: + await mock.reset() + cc = await b.chat( + IMAGE_GA, + EDIT, + params=LEGACY, + chat_id=cx1.chat_id, + parent_id=cg.message_id, + ) + outsider_calls = len(await mock.requests(_gen)) + + # the pipe's own check (the filter hands it the chat) + for key, client, target in ( + ("participant", part, _target(cb)), + ("admin", t.owui, _target(cb)), + ("viewer", users["viewer"], _target(cy2)), + ("outsider", users["outsider"], _target(cb)), + ): + redirected[key] = await _api_edit(client, mock, f"{EDIT} {target}") + mark = t.mark() + chat_fault, chat_fault_req = await _api_edit( + part, mock, f"{EDIT} {_target(cb)} e2e-chat-fault={cb.chat_id}" + ) + await t.log.settle(0.5) + chat_fault_logged = len(t.log.lines(mark, "e2e chat access fault")) + + # files named in the message: Open WebUI's rule per file + url = "/api/v1/files/{}/content" + mark = t.mark() + links, links_req = await _api_edit( + part, + mock, + f"Edit these ![a]({url.format(upload['id'])}) " + f"![b]({url.format(admin_file['id'])}) ![c]({url.format(private['id'])})", + ) + await t.log.settle(0.5) + denied = len(t.log.lines(mark, FILE_REFUSED)) + ids_logged = [ + line + for line in t.log.lines(mark, admin_file["id"], private["id"]) + if "function_gemini" in line + ] + mark = t.mark() + file_fault, file_fault_req = await _api_edit( + part, + mock, + f"Edit this ![a]({url.format(upload['id'])}) e2e-file-fault={upload['id']}", + ) + await t.log.settle(0.5) + file_fault_logged = len( + [ + line + for line in t.log.lines(mark, "File access check failed") + if "RuntimeError" in line + ] + ) + ids_logged += [ + line + for line in t.log.lines(mark, upload["id"]) + if "function_gemini" in line + ] + t.expect_warnings(mark, ("function_gemini", "File access check failed")) + await mock.reset() + video = await part.chat( + VEO, + f"Animate this ![frame]({url.format(upload['id'])})", + stream=False, + timeout=180, + ) + video_image = await _video_image(mock) + except Exception as exc: # noqa: BLE001 - reported in the checks + error = f"setup failed: {exc!r} " + finally: + for client in users.values(): + await client.close() + await t.owui.delete_model(IMAGE_GA) + await t.owui.delete_model(VEO) + await t.owui.api("DELETE", f"/api/v1/functions/id/{REDIRECT_FILTER_ID}/delete") + if group_id: + await t.owui.api("DELETE", f"/api/v1/groups/id/{group_id}/delete") + setup = ( + f"{error}grants={grants} shares={shares} group_add={group_status} " + f"folder={folder} turn1: {_brief(cx1)} files={len(getattr(cx1, 'files', []))}" + ) + ready = not error and grants == [200, 200, 200, True, True] and shares == [200] * 4 + + t.check( + "shared.visible", + "Open WebUI lets the participant read the owner's upload and the " + "generated image of chat X (the pipe links its images to the chat)", + ready and visible.get("upload") == 200 and visible.get("generated") == [200], + f"{setup} participant: upload HTTP {visible.get('upload')} " + f"generated HTTP {visible.get('generated')}", + ) + t.check( + "shared.participant", + "browser, a participant (user grant, Allow replies) edits in the owner's " + "chat: the owner's upload and the generated image of turn 1 are sent", + ready + and getattr(cb, "done", False) + and rb["sent"] == [f"{EDIT} final-image-2", *WITH_HISTORY[1:]], + f"{setup} participant: {_brief(cb)} {_edit_report(rb)}", + ) + t.check( + "shared.owner", + "browser, the owner continues after the participant: the participant's " + "generated image is sent too", + ready + and getattr(ca, "done", False) + and ra["images"] == ["upload1", "final", "gen2"], + f"{setup} owner: {_brief(ca)} {_edit_report(ra)}", + ) + t.check( + "shared.group", + "browser, a member of a group the chat is shared with edits in it: every " + "image of the chat is sent", + ready + and group_status == 200 + and getattr(cg, "done", False) + and rg["images"] == ["upload1", "final", "gen2", "gen3"], + f"{setup} member: {_brief(cg)} {_edit_report(rg)}", + ) + t.check( + "shared.refused", + "browser, a viewer of a Clone only share (also with read-only folder " + "access) and a user the chat is not shared with: Open WebUI refuses " + "their turns (HTTP 404), nothing reaches Gemini", + ready + and getattr(cd, "http_status", None) == 404 + and getattr(cc, "http_status", None) == 404 + and viewer_calls == 0 + and outsider_calls == 0, + f"{setup} viewer: {_brief(cd)} upstream={viewer_calls} " + f"outsider: {_brief(cc)} upstream={outsider_calls}", + ) + for key, title, expected in ( + ( + "participant", + "a participant: the chat's images are sent (the filter works)", + WITH_HISTORY, + ), + ( + "admin", + "an admin (read access to every chat, not write): nothing of the " + "chat is read", + [EDIT], + ), + ( + "viewer", + "a viewer with read access only (Clone only share, read-only " + "folder): nothing of the chat is read", + [EDIT], + ), + ( + "outsider", + "a user the chat is not shared with: nothing of the chat is read", + [EDIT], + ), + ): + result, req = redirected.get(key, (None, empty)) + t.check( + f"shared.pipe-{key}", + "API, the pipe called for a saved chat without Open WebUI's check " + f"(test filter): {title}", + ready + and getattr(result, "status", None) == 200 + and req["sent"] == expected, + f"{setup} {key}: HTTP {getattr(result, 'status', None)} {_edit_report(req)}", + ) + t.check( + "shared.chat-fault", + "API, the chat access check raises for a participant: the chat is not " + "read (logged), the answer completes", + ready + and getattr(chat_fault, "status", None) == 200 + and chat_fault_req["sent"] == [EDIT] + and chat_fault_logged > 0, + f"{setup} HTTP {getattr(chat_fault, 'status', None)} " + f"{_edit_report(chat_fault_req)} logged={chat_fault_logged}", + ) + t.check( + "shared.links", + "API, markdown links of a participant: the owner's upload of the shared " + "chat is read, a file of the admin and a file of the owner outside the " + "chat are not (logged without their ids)", + ready + and getattr(links, "status", None) == 200 + and links_req["images"] == ["upload1"] + and denied == 2 + and not ids_logged, + f"{setup} HTTP {getattr(links, 'status', None)} {_edit_report(links_req)} " + f"denied_logged={denied} ids_logged={[short(x, 120) for x in ids_logged]}", + ) + t.check( + "shared.file-fault", + "API, the file access check raises for a participant's link to a file " + "of the shared chat: the file is not read (logged without its id)", + ready + and getattr(file_fault, "status", None) == 200 + and file_fault_req["images"] == [] + and file_fault_logged > 0 + and not ids_logged, + f"{setup} HTTP {getattr(file_fault, 'status', None)} " + f"{_edit_report(file_fault_req)} logged={file_fault_logged}", + ) + t.check( + "shared.video", + "Veo image-to-video, API, a participant links the owner's upload of the " + "shared chat: it is sent as instances[0].image", + ready and getattr(video, "status", None) == 200 and video_image == "upload1", + f"{setup} HTTP {getattr(video, 'status', None)} image={video_image!r}", + ) diff --git a/tests/e2e/suites/gemini.py b/tests/e2e/suites/gemini.py index a5ab96c..a214692 100644 --- a/tests/e2e/suites/gemini.py +++ b/tests/e2e/suites/gemini.py @@ -41,6 +41,9 @@ images, older saved forms (data: URL files, markdown links), temporary chats and database errors use the request, files of another user are read only for an admin (also on the Veo path) + shared chats shared with "Allow replies" (Open WebUI 0.12): participants + edit with the chat's images, access as Open WebUI decides it + (suites/_gemini_shared.py) toolsapi native tool calling, API path: client tools -> tool_calls (stream, non-stream, finish_reason), text answers, streaming valve off, malformed / unexpected calls, continuation with signatures and @@ -100,6 +103,7 @@ "concurrency", "streamimg", "imgedit", + "shared", "toolsapi", "tools", "owuitools", @@ -179,6 +183,9 @@ "Dropping unmatched tool result", "Could not restore stored model content", "Invalid tool call arguments", + # shared chats (1.20.0) + "Failed to link generated image files", + "File access check failed", ) ) # Valves and UserValves of google_gemini.py 1.16.1 with their defaults (valve @@ -537,6 +544,7 @@ async def run(t: Suite) -> None: ("concurrency", concurrency), ("streamimg", streamimg), ("imgedit", imgedit), + ("shared", shared_group), ("toolsapi", toolsapi_group), ("tools", tools_group), ("owuitools", owuitools_group), @@ -2986,6 +2994,13 @@ async def tools_group(t: Suite, mock) -> None: await _gemini_tools.tools(t, mock) +# Chats shared with "Allow replies" (Open WebUI 0.12): suites/_gemini_shared.py +async def shared_group(t: Suite, mock) -> None: + from . import _gemini_shared + + await _gemini_shared.shared(t, mock) + + # Open WebUI's built-in image and code interpreter tools: suites/_gemini_owuitools.py async def owuitools_group(t: Suite, mock) -> None: from . import _gemini_owuitools From 0b4c4175749ed99a721cc83ceefe36021c44c785 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:47:52 +0200 Subject: [PATCH 04/12] test(e2e): gemini.owui012 group for the Open WebUI 0.12 features New group in suites/_gemini_owui012.py (12 checks, version-gated: on Open WebUI 0.11 a check of a missing feature passes with "feature not in Open WebUI " and checks what that version does instead): - compaction, compaction-followup: compaction between tool rounds with a threshold of 1 token; the summary request has no tools and the next request holds the summary, the user message and the kept rounds with their signatures; in a follow-up turn whose boundary falls into the earlier turn Open WebUI moves the user message before that turn's kept round (kinds= in the detail), the request stays valid - summary-error, summary-empty (between tool rounds) and saved-summary (before a reply, also on 0.11): a failed (HTTP 400) or empty summary request leaves Open WebUI's fallback summary, not the pipe's error or placeholder text, also in the summary saved with the chat - toolsearch, toolsearch-followup, toolsearch-mapped: search_tools and the list, the found tool declared and run in the next round, in the next turn, and an OpenAPI tool whose name needs mapping - controls: 0.12 refuses model controls on a workspace model of a Gemini pipe model (HTTP 400); custom-params: the custom params a control would set reach the pipe from a workspace model and from the chat's Controls - stream-chat, stream-model: tool turns with stream=false per chat and per model (Open WebUI sets stream=false for the model) mock_gemini.py answers Open WebUI's compaction prompt with "MOCK-SUMMARY of " of the compacted section; triggers summary-400 and summary-empty affect only that request. With google_gemini.py 1.19.1 six checks fail on 0.12.0 (summary-error, summary-empty, saved-summary and the three toolsearch checks); with 1.20.0 all pass on 0.12.0 and 0.11.4. +semver: none Co-Authored-By: Claude Opus 5.5 --- tests/e2e/mocks/mock_gemini.py | 57 ++- tests/e2e/suites/_gemini_owui012.py | 706 ++++++++++++++++++++++++++++ tests/e2e/suites/gemini.py | 15 + 3 files changed, 774 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/suites/_gemini_owui012.py diff --git a/tests/e2e/mocks/mock_gemini.py b/tests/e2e/mocks/mock_gemini.py index 899dc3d..2f7c3fb 100644 --- a/tests/e2e/mocks/mock_gemini.py +++ b/tests/e2e/mocks/mock_gemini.py @@ -75,6 +75,16 @@ slow-thinking streaming text models: twelve thought parts "Slow thought ." 0.5 s apart, then the answer "Slow answer." +Open WebUI's context compaction prompt ("Summarize the conversation history +that will be compacted ..."; before a reply and, from Open WebUI 0.12, between +tool rounds) is answered with "MOCK-SUMMARY of ", the mock's call ids +(mock-call--) in its "Messages Being Compacted" section in order ("no +calls" without any). Triggers in that prompt (the chat's messages are part of +it, so a word in the user message reaches only the summary request): + summary-400 HTTP 400 INVALID_ARGUMENT for the summary request + summary-empty a summary without text: the thought part only (when + requested), finishReason MAX_TOKENS + Native tool calling: ``MOCKTOOLS:{json}`` in the turn's user message {"rounds": [[{"name": "get_current_timestamp", "args": {}}], [...]], "text_before": "Let me check.", "split": false, "noid": false, "nosig": false, @@ -120,8 +130,8 @@ ``fc_mode``, ``allowed_names``, ``include_flag``, ``kinds`` (role and part kinds per content), ``fc`` / ``fr`` (function call / response parts with content index, id, name, args / response; ``sig`` of a call is none, skip, issued or -bad), ``round``, ``directive``, ``answer`` (fc, final, text, malformed, -mock-error, http-), ``issued``, ``server_echoed``, +bad), ``round``, ``directive``, ``answer`` (fc, final, text, summary, +malformed, mock-error, http-), ``issued``, ``server_echoed``, ``synthetic_ids_upstream`` and ``orphan_fr``. usage: python mock_gemini.py [--port 9101] @@ -199,6 +209,12 @@ ), } STREAM_PAUSE = 0.05 # seconds after every streamed chunk +# Open WebUI's context compaction prompt (DEFAULT_CONTEXT_COMPACTION_PROMPT) +COMPACTION_TASK = "Summarize the conversation history that will be compacted" +COMPACTED_SECTION = re.compile( + r"### Messages Being Compacted:(.*?)(?:### Recent Messages Kept In Context:|$)", + re.DOTALL, +) def png_b64(r: int, g: int, b: int) -> str: @@ -771,9 +787,35 @@ def _first_since_reset(request: web.Request, trigger: str) -> bool: return len(seen) == 1 +def _compaction_summary(text: str) -> Optional[str]: + """The answer to Open WebUI's context compaction prompt (None for other + prompts): "MOCK-SUMMARY of" the mock's call ids in the messages being + compacted, in order.""" + if COMPACTION_TASK not in text: + return None + match = COMPACTED_SECTION.search(text) + ids = re.findall(r"mock-call-\d+-\d+", match.group(1) if match else "") + return "MOCK-SUMMARY of " + (",".join(dict.fromkeys(ids)) or "no calls") + + +def _summary_answer(model: str, body, summary: str, usage: dict) -> dict: + """GenerateContentResponse for a compaction prompt (``summary-empty``: no + text, finishReason MAX_TOKENS).""" + parts = ( + [{"text": "Summary thinking.", "thought": True}] + if _thoughts(model, body) + else [] + ) + if "summary-empty" in _turn_text(body): + return _chunk(parts, body, True, usage, "MAX_TOKENS") + return _chunk(parts + [{"text": summary}], body, True, usage) + + def _reject(request: web.Request, model: str, body: dict) -> Optional[web.Response]: """Errors on purpose (triggers) and tools a model does not support (400).""" text = _turn_text(body) + if "summary-400" in text and _compaction_summary(text) is not None: + return _error(400, "mock: summary-400", "INVALID_ARGUMENT") if "force-400" in text: return _error(400, "mock: force-400", "INVALID_ARGUMENT") if "force-500-once" in text: @@ -873,6 +915,9 @@ def _answer(model: str, body) -> dict: if blocked: return blocked thoughts = _thoughts(model, body) + summary = _compaction_summary(text) + if summary is not None: + return _summary_answer(model, body, summary, USAGE_TEXT) task = task_answer(text) if task: parts = [{"text": "Task thinking.", "thought": True}] if thoughts else [] @@ -916,6 +961,9 @@ def _stream_chunks(model: str, body) -> list: _chunk([{"text": "Here is your image."}], body, True, USAGE_IMAGE) ) return chunks + summary = _compaction_summary(text) + if summary is not None: + return [_summary_answer(model, body, summary, USAGE_STREAM)] task = task_answer(text) paced = next((key for key in PACED if key in text), None) if paced and not task: @@ -973,11 +1021,12 @@ async def model_action(request: web.Request) -> web.StreamResponse: annotate(request, answer=f"http-{rejected.status}") return rejected text = _turn_text(body) - if not (task_answer(text) or _blocked(text)): + summary = _compaction_summary(text) is not None + if not (summary or task_answer(text) or _blocked(text)): plan = _tool_plan(model, view) annotate( request, - answer=plan["answer"] if plan else "text", + answer="summary" if summary else plan["answer"] if plan else "text", issued=(plan or {}).get("issued"), ) if action == "generateContent": diff --git a/tests/e2e/suites/_gemini_owui012.py b/tests/e2e/suites/_gemini_owui012.py new file mode 100644 index 0000000..c932541 --- /dev/null +++ b/tests/e2e/suites/_gemini_owui012.py @@ -0,0 +1,706 @@ +""" +Open WebUI 0.12 features around the Gemini pipe's native tool calling: group +``owui012`` of the gemini suite (run by suites/gemini.py; the leading +underscore keeps this module out of the suite discovery). + +- context compaction (Admin Settings -> Interface): from 0.12 Open WebUI also + compacts between tool rounds (``compaction``, ``compaction-followup``); the + summary comes from the chat's model, i.e. the pipe with ``__task__`` = + ``context_compaction``, and a failed or empty summary must not become the + conversation summary (``summary-error``, ``summary-empty`` between tool + rounds, ``saved-summary`` before a reply, which Open WebUI 0.11 does too) +- Tool Search (experimental, 0.12): deferred tools are left out of + body["tools"]; ``search_tools`` returns their definitions and the model calls + them by name (``toolsearch``, ``toolsearch-mapped``, ``toolsearch-followup``) +- model controls (0.12): refused for pipe models (``controls``); the custom + parameters a control would set reach the pipe as Advanced Params of a + workspace model and of the chat (``custom-params``) +- Stream Chat Response per chat (honoured by the 0.12 web UI) and per model: + a tool turn with ``stream=false`` (``stream-chat``, ``stream-model``) + +Checks of a feature that the tested Open WebUI does not have pass with +"feature not in Open WebUI " in their title and check what that +version does instead. Every detail starts with ``mock_ok=`` (the preflight of +the tool groups) and ``owui=``. +""" + +import json + +from harness import Suite, short +from harness.known import version_tuple + +from ._gemini_tools import ( + CALC, + LOOKUP, + OPENAPI_ID, + TIMESTAMP, + Ctx, + _answers, + _bdetail, + _calls, + _connections, + _declared, + _fcs, + _final, + _frs, + _req_tokens, + directive, + gemini_function_name, + preflight, +) +from .gemini import PRO3, TEXT, _body, _gc, _gen, _last_text + +CALC2 = {"name": "calculate_timestamp", "args": {"days_ago": 2}} +SUMMARY_HEADER = "[CONVERSATION SUMMARY]" +# Open WebUI's fallback summary when the model gave none (_generate_summary): +# "- : " per compacted message +FALLBACK_USER = "- user: Use the tools." +# Chat settings the group changes (restored at its end) +COMPACTION = dict( + ENABLE_CONTEXT_COMPACTION=True, + CONTEXT_COMPACTION_MODEL="", + CONTEXT_COMPACTION_TOKEN_THRESHOLD=1, + CONTEXT_COMPACTION_TOKEN_CAP=1, +) +TOOL_SEARCH = dict( + ENABLE_TOOL_SEARCH=True, + TOOL_SEARCH_DEFER_THRESHOLD=0, + TOOL_SEARCH_ALWAYS_LOADED=["get_current_timestamp"], + TOOL_SEARCH_DEFER_BUILTIN_TOOLS=True, +) +OFF = dict(ENABLE_CONTEXT_COMPACTION=False, ENABLE_TOOL_SEARCH=False) +# Workspace models of the group (deleted at its end) +PARAMS_MODEL = "e2e-g012-params" +PARAMS_MODEL_25 = "e2e-g012-params25" +NOSTREAM_MODEL = "e2e-g012-nostream" +CONTROLS_MODEL = "e2e-g012-controls" +# A Thinking control as the 0.12 model editor saves it +THINKING_CONTROL = { + "thinking": { + "label": "Thinking", + "default": "high", + "options": { + "low": {"label": "Low", "params": {"reasoning_effort": "low"}}, + "high": {"label": "High", "params": {"reasoning_effort": "high"}}, + }, + } +} + + +def search(query: str) -> dict: + """A call of Open WebUI's search_tools (Tool Search).""" + return {"name": "search_tools", "args": {"query": query}} + + +def _system(entry: dict) -> str: + """Text of a request's system instruction.""" + instruction = _body(entry).get("systemInstruction") + if isinstance(instruction, str): + return instruction + parts = instruction.get("parts") if isinstance(instruction, dict) else None + return "".join( + p["text"] + for p in parts or [] + if isinstance(p, dict) and isinstance(p.get("text"), str) + ) + + +def _summary(entry: dict) -> str: + """The conversation summary in a request's system instruction ('' if none).""" + text = _system(entry) + return text.split(SUMMARY_HEADER, 1)[1].strip() if SUMMARY_HEADER in text else "" + + +def _split(reqs: list) -> tuple: + """(turn requests, summary requests) of a recorded tool turn.""" + summaries = [e for e in reqs if e.get("answer") == "summary"] + failed = [ + e + for e in reqs + if str(e.get("answer", "")).startswith("http-") + and "Summarize the conversation history" in _last_text(e) + ] + turns = [e for e in reqs if e not in summaries and e not in failed] + return turns, summaries + failed + + +def _events(chat, kind: str) -> list: + return [ + (e.get("data") or {}).get("description") + for e in chat.events + if isinstance(e, dict) and e.get("type") == kind + ] + + +def _thinking(entry: dict) -> dict: + return _gc(entry).get("thinkingConfig") or {} + + +async def _upsert(t: Suite, model_id: str, base: str, params: dict) -> tuple: + """Create (or update) a workspace model on a pipe model; (status, body).""" + form = { + "id": model_id, + "base_model_id": base, + "name": model_id, + "params": params, + "meta": {"description": "e2e owui012", "capabilities": {}}, + } + status, existing = await t.owui.api("GET", f"/api/v1/models/model?id={model_id}") + if status == 200 and isinstance(existing, dict) and existing.get("id"): + result = await t.owui.api("POST", "/api/v1/models/model/update", form) + else: + result = await t.owui.api("POST", "/api/v1/models/create", form) + await t.owui.models(refresh=True) + return result + + +# ==================================================================== group +async def owui012(t: Suite, mock) -> None: + ctx = await preflight(t, mock) + version = await t.owui.version() + v012 = version_tuple(version) >= (0, 12) + config = await t.owui.chat_config() + tool_search = "ENABLE_TOOL_SEARCH" in config + restore = { + key: config[key] for key in {**COMPACTION, **TOOL_SEARCH} if key in config + } + ctx.version = version + previous_servers = None + try: + async with t.browser() as b: + await t.owui.set_chat_config(**COMPACTION) + await compaction(ctx, b, v012) + await compaction_followup(ctx, b, v012) + await summary_failed(ctx, b, v012, "summary-error") + await summary_failed(ctx, b, v012, "summary-empty") + await saved_summary(ctx, b) + await t.owui.set_chat_config(**OFF) + previous_servers = await t.owui.set_tool_servers(_connections()[:1]) + await t.owui.set_chat_config(**TOOL_SEARCH) + await toolsearch(ctx, b, tool_search) + await t.owui.set_chat_config(**OFF) + await controls(ctx, v012) + await custom_params(ctx, b) + await stream_chat(ctx, b) + await stream_model(ctx, b) + finally: + await t.owui.set_chat_config(**restore) + if previous_servers is not None: + await t.owui.set_tool_servers(previous_servers) + for model_id in (PARAMS_MODEL, PARAMS_MODEL_25, NOSTREAM_MODEL, CONTROLS_MODEL): + await t.owui.delete_model(model_id) + + +def _check(ctx: Ctx, sid: str, title: str, ok, detail: str) -> bool: + return ctx.check( + f"owui012.{sid}", title, ok, f"owui={getattr(ctx, 'version', '?')} {detail}" + ) + + +def _absent(ctx: Ctx, feature: str) -> str: + return f"feature not in Open WebUI {getattr(ctx, 'version', '?')}: {feature}" + + +# =============================================================== compaction +async def compaction(ctx: Ctx, b, v012: bool) -> None: + """Three tool rounds with compaction on and a threshold of 1 token: from + 0.12 Open WebUI compacts before the 3rd and 4th request of the turn.""" + await ctx.mock.reset() + c = await ctx.chat(b, PRO3, directive([[TIMESTAMP], [CALC], [CALC2]])) + reqs = await ctx.requests() + turns, summaries = _split(reqs) + r2, r3 = (turns + [{}, {}, {}, {}])[2:4] + statuses = [e.get("status") for e in reqs] + saved = ( + c.done + and not c.error + and [s for _, s, _ in _calls(c)] == ["completed"] * 3 + and len(c.function_outputs) == 3 + ) + if v012: + ok = ( + saved + and statuses == [200] * len(reqs) + and _answers(turns) == "[fc,fc,fc,final]" + and [e.get("answer") for e in summaries] == ["summary", "summary"] + and all(not _declared(e) and not e.get("tool_kinds") for e in summaries) + and r2.get("kinds") == ["user:text", "model:fc", "user:fr"] + and _fcs(r2) == [("mock-call-1-0", "calculate_timestamp", "issued")] + and [i for i, _, _ in _frs(r2)] == ["mock-call-1-0"] + and not r2.get("orphan_fr") + and _summary(r2) == "MOCK-SUMMARY of mock-call-0-0" + and r3.get("kinds") == ["user:text", "model:fc", "user:fr"] + and _fcs(r3) == [("mock-call-2-0", "calculate_timestamp", "issued")] + and _summary(r3) == "MOCK-SUMMARY of mock-call-1-0" + # the final answer echoes the responses Gemini got: the last round's + and _final(c).startswith("MOCK-FINAL calculate_timestamp=") + and _final(c).count("_timestamp=") == 1 + ) + title = ( + "compaction between tool rounds (3 rounds, threshold 1): the summary " + "request goes through the pipe without tools and answers a plain " + "summary; after it the pipe gets the summary in the system prompt, the " + "user message and the last round only, and sends a valid request " + "(function call with its signature + its response, no orphan)" + ) + else: + ok = ( + saved + and _answers(turns) == "[fc,fc,fc,final]" + and not summaries + and len(r3.get("fc") or []) == 3 + and not _summary(r3) + ) + title = _absent( + ctx, + "no compaction between tool rounds (only before a reply); the 3-round " + "turn with compaction on keeps every round", + ) + _check( + ctx, + "compaction", + title, + ok, + _bdetail( + c, + turns, + f"statuses={statuses} summaries={[e.get('answer') for e in summaries]} " + f"summary_r2={short(_summary(r2), 80)!r} summary_r3=" + f"{short(_summary(r3), 80)!r} r2: {_req_tokens(r2)} r3: " + f"{_req_tokens(r3)} events={_events(c, 'context_compaction')}", + ), + ) + + +async def compaction_followup(ctx: Ctx, b, v012: bool) -> None: + """A tool turn after a turn with two rounds: the compaction boundary falls + into the earlier turn, Open WebUI puts the current user message before the + kept rounds of that turn. The request must stay valid.""" + await ctx.t.owui.set_chat_config(ENABLE_CONTEXT_COMPACTION=False) + try: + await ctx.mock.reset() + first = await ctx.chat(b, PRO3, directive([[TIMESTAMP], [CALC]])) + finally: + await ctx.t.owui.set_chat_config(**COMPACTION) + if not first.chat_id: + _check(ctx, "compaction-followup", "follow-up turn", False, "turn 1 failed") + return + await ctx.mock.reset() + text = directive([[CALC2], [TIMESTAMP]]) + c = await ctx.chat(b, PRO3, text, chat_id=first.chat_id, parent_id=first.message_id) + reqs = await ctx.requests() + turns, summaries = _split(reqs) + after = turns[1] if len(turns) > 1 else {} + kinds = after.get("kinds") or [] + statuses = [e.get("status") for e in reqs] + first_sigs = [ + s for x in after.get("fc") or [] if x.get("first") for s in [x["sig"]] + ] + valid = ( + c.done + and not c.error + and statuses == [200] * len(reqs) + and not after.get("orphan_fr") + and len(after.get("fc") or []) == len(after.get("fr") or []) + and all(sig == "issued" for sig in first_sigs) + ) + if v012: + ok = ( + valid and len(summaries) >= 1 and _summary(after).startswith("MOCK-SUMMARY") + ) + title = ( + "compaction in a follow-up tool turn whose boundary falls into the " + "earlier turn: the request after it is valid (every call with its " + "response, first calls signed); kinds= shows the order Open WebUI sends" + ) + else: + ok = valid and not summaries and not _summary(after) + title = _absent(ctx, "no compaction between tool rounds; the turn completes") + _check( + ctx, + "compaction-followup", + title, + ok, + _bdetail( + c, + turns, + f"statuses={statuses} summaries={len(summaries)} prompt_first=" + f"{kinds[:1] == ['user:text'] and kinds.count('user:text') == 1} " + f"summary={short(_summary(after), 80)!r} {_req_tokens(after)}", + ), + ) + + +async def summary_failed(ctx: Ctx, b, v012: bool, trigger: str) -> None: + """The summary request between tool rounds fails (HTTP 400) or returns no + text (thinking only, MAX_TOKENS): the conversation summary must not be the + pipe's error or placeholder text, Open WebUI then uses its own summary of + the compacted messages.""" + mark = ctx.t.mark() + await ctx.mock.reset() + word = "summary-400" if trigger == "summary-error" else trigger + c = await ctx.chat(b, PRO3, directive([[TIMESTAMP], [CALC]]) + f" {word}") + reqs = await ctx.requests() + if trigger == "summary-error": + ctx.t.expect_errors(mark, ("function_gemini", "mock: summary-400")) + turns, summaries = _split(reqs) + r2 = turns[2] if len(turns) > 2 else {} + summary = _summary(r2) + expected = "http-400" if trigger == "summary-error" else "summary" + leaked = [ + text + for text in ( + "Error", + "INVALID_ARGUMENT", + "mock: summary-400", + "[No content generated", + "Summary thinking.", + ) + if text in summary + ] + if v012: + ok = ( + c.done + and not c.error + and _answers(turns) == "[fc,fc,final]" + and [e.get("answer") for e in summaries] == [expected] + and summary.startswith(FALLBACK_USER) + and not leaked + ) + what = "fails (HTTP 400)" if trigger == "summary-error" else "has no text" + title = ( + f"compaction between tool rounds, the summary request {what}: the " + "next request carries Open WebUI's own summary of the compacted " + "messages, not the pipe's error or placeholder text" + ) + else: + ok = c.done and not summaries and not summary + title = _absent(ctx, "no compaction between tool rounds") + _check( + ctx, + trigger, + title, + ok, + _bdetail( + c, + turns, + f"summaries={[e.get('answer') for e in summaries]} leaked={leaked} " + f"summary={short(summary, 160)!r}", + ), + ) + + +async def saved_summary(ctx: Ctx, b) -> None: + """Compaction before a reply (Open WebUI 0.11 and 0.12): the summary is + saved with the chat and replaces the compacted turns in every later + request. A failed summary request must not save the pipe's error text.""" + mark = ctx.t.mark() + turns, chat_id, parent = [], None, None + for i, text in enumerate( + ("First turn.", "Second turn.", "Third turn summary-400.") + ): + if i == 2: + await ctx.mock.reset() + c = await ctx.chat(b, TEXT, text, chat_id=chat_id, parent_id=parent) + turns.append(c) + chat_id, parent = c.chat_id, c.message_id + if not c.done: + break + reqs = await ctx.requests() + ctx.t.expect_errors(mark, ("function_gemini", "mock: summary-400")) + main, summaries = _split(reqs) + saved = await ctx.t.owui.get_chat(chat_id) if chat_id else {} + messages = ((saved.get("chat") or {}).get("history") or {}).get("messages") or {} + stored = [ + m.get("contextSummary") + for m in messages.values() + if m.get("role") == "user" and m.get("contextSummary") + ] + sent = _summary(main[-1]) if main else "" + leaked = [ + text + for text in ("Error", "INVALID_ARGUMENT", "mock: summary-400") + if any(text in s for s in stored + [sent]) + ] + ok = ( + len(turns) == 3 + and all(c.done for c in turns) + and [e.get("answer") for e in summaries] == ["http-400"] + and len(stored) == 1 + and stored[0].startswith("- user: First turn.") + and sent == stored[0] + and not leaked + ) + _check( + ctx, + "saved-summary", + "compaction before a reply (third turn), the summary request fails " + "(HTTP 400): the summary Open WebUI saves with the chat and sends is its " + "own summary of the compacted turns, not the pipe's error text", + ok, + f"done={[c.done for c in turns]} summaries=" + f"{[e.get('answer') for e in summaries]} stored={[short(s, 120) for s in stored]} " + f"sent={short(sent, 120)!r} leaked={leaked} upstream={_answers(main)}", + ) + + +# ============================================================== tool search +async def toolsearch(ctx: Ctx, b, available: bool) -> None: + """Tool Search with every tool deferred except get_current_timestamp.""" + await ctx.mock.reset() + c = await ctx.chat(b, PRO3, directive([[search("calculate_timestamp")], [CALC]])) + reqs = await ctx.requests() + r0, r1 = (reqs + [{}, {}])[:2] + declared0, declared1 = _declared(r0), _declared(r1) + manifest = "" in _system(r0) + found = [x for x in r1.get("fr") or [] if x.get("name") == "search_tools"] + if available: + ok = ( + c.done + and not c.error + and {"search_tools", "get_current_timestamp"} <= set(declared0) + and "calculate_timestamp" not in declared0 + and manifest + and "calculate_timestamp" in _system(r0) + and "calculate_timestamp" in declared1 + and len(found) == 1 + and "calculate_timestamp" in json.dumps(found[0].get("response")) + and _answers(reqs) == "[fc,fc,final]" + and _calls(c) + == [ + ("search_tools", "completed", "mock-call-0-0"), + ("calculate_timestamp", "completed", "mock-call-1-0"), + ] + and "calculate_timestamp=" in _final(c) + ) + title = ( + "Tool Search: search_tools and the always loaded tool declared, the " + "deferred calculate_timestamp only listed in ; after " + "search_tools returned it, it is declared in the next round, called and " + "run by Open WebUI" + ) + else: + ok = c.done and "search_tools" not in declared0 and not manifest + title = _absent( + ctx, "Tool Search: search_tools is not declared, no list" + ) + _check( + ctx, + "toolsearch", + title, + ok, + _bdetail( + c, + reqs, + f"manifest={manifest} declared0_n={len(declared0)} " + f"search_declared={'search_tools' in declared0} " + f"found_declared={'calculate_timestamp' in declared1} {_req_tokens(r1)}", + ), + ) + await toolsearch_followup(ctx, b, c, available) + await toolsearch_mapped(ctx, b, available) + + +async def toolsearch_followup(ctx: Ctx, b, first, available: bool) -> None: + """The next turn calls the tool found in turn 1 without searching again + (without Tool Search the tool is declared anyway).""" + await ctx.mock.reset() + c = await ctx.chat( + b, + PRO3, + directive([[CALC2]]), + chat_id=first.chat_id, + parent_id=first.message_id, + ) + reqs = await ctx.requests() + r0 = reqs[0] if reqs else {} + _check( + ctx, + "toolsearch-followup", + "Tool Search, next turn: the tool search_tools returned in turn 1 is " + "declared again (its result is in the history), the direct call runs" + if available + else _absent(ctx, "Tool Search: every tool is declared, the direct call runs"), + bool(first.chat_id) + and c.done + and not c.error + and "calculate_timestamp" in _declared(r0) + and _answers(reqs) == "[fc,final]" + and _calls(c) == [("calculate_timestamp", "completed", "mock-call-0-0")], + _bdetail(c, reqs, _req_tokens(reqs[-1] if reqs else {})), + ) + + +async def toolsearch_mapped(ctx: Ctx, b, available: bool) -> None: + """A deferred OpenAPI tool whose name Gemini does not accept (lookup.v2).""" + tool_mock = ctx.t.mock("tools") + await ctx.mock.reset() + await tool_mock.reset() + call = {"name": LOOKUP, "args": {"key": "k2"}} + c = await ctx.chat( + b, + PRO3, + directive([[search(LOOKUP)], [call]] if available else [[call]]), + tool_ids=[f"server:{OPENAPI_ID}"], + ) + reqs = await ctx.requests() + log = await tool_mock.requests() + lookup = [e.get("query", {}).get("key") for e in log if e.get("path") == "/lookup"] + mapped = gemini_function_name(LOOKUP) + r0, r1 = (reqs + [{}, {}])[:2] + if available: + ok = mapped not in _declared(r0) and mapped in _declared(r1) + call_id = "mock-call-1-0" + title = ( + f"Tool Search, a deferred OpenAPI tool ({LOOKUP}): declared as {mapped} " + "after search_tools found it, called under its own name, the tool " + "server gets the call" + ) + else: + ok = mapped in _declared(r0) + call_id = "mock-call-0-0" + title = _absent( + ctx, + f"Tool Search: the OpenAPI tool {LOOKUP} is declared as {mapped} in " + "the first request and called", + ) + _check( + ctx, + "toolsearch-mapped", + title, + ok + and c.done + and not c.error + and lookup == ["k2"] + and (LOOKUP, "completed", call_id) in _calls(c), + _bdetail( + c, + reqs, + f"lookup={lookup} mapped_declared=[{mapped in _declared(r0)}," + f"{mapped in _declared(r1)}]", + ), + ) + + +# ======================================================== model controls etc. +async def controls(ctx: Ctx, v012: bool) -> None: + """Model controls (0.12) on a workspace model of a Gemini pipe model.""" + status, data = await _upsert( + ctx.t, CONTROLS_MODEL, PRO3, {"model_controls": THINKING_CONTROL} + ) + detail = json.dumps(data)[:200] if not isinstance(data, str) else data[:200] + if v012: + ok = status == 400 and "server-managed provider model" in detail + title = ( + "model controls on a workspace model of a Gemini pipe model: Open " + "WebUI 0.12 refuses them (HTTP 400), so no control reaches the pipe" + ) + else: + ok = status == 200 + title = _absent(ctx, "model controls (model_controls is an ordinary param)") + await ctx.t.owui.delete_model(CONTROLS_MODEL) + _check(ctx, "controls", title, ok, f"http={status} answer={detail}") + + +async def custom_params(ctx: Ctx, b) -> None: + """The custom parameters a Thinking control would set, as Advanced Params + of a workspace model and of the chat (Controls).""" + s1, _ = await _upsert( + ctx.t, + PARAMS_MODEL, + PRO3, + {"custom_params": {"reasoning_effort": "low", "include_thoughts": "false"}}, + ) + s2, _ = await _upsert( + ctx.t, PARAMS_MODEL_25, TEXT, {"custom_params": {"thinking_budget": "1024"}} + ) + await ctx.mock.reset() + r1 = await ctx.t.owui.chat(PARAMS_MODEL, "Hi", stream=False) + think1 = _thinking(await ctx.mock.last(_gen)) + await ctx.mock.reset() + r2 = await ctx.t.owui.chat(PARAMS_MODEL_25, "Hi", stream=False) + think2 = _thinking(await ctx.mock.last(_gen)) + await ctx.mock.reset() + c = await ctx.chat( + b, PARAMS_MODEL, "Hi", params={"custom_params": {"reasoning_effort": "high"}} + ) + think3 = _thinking(await ctx.mock.last(_gen)) + # informational: reasoning_effort on Gemini 2.5 (not mapped to a budget) + await ctx.mock.reset() + await ctx.t.owui.chat(TEXT, "Hi", stream=False, reasoning_effort="low") + think4 = _thinking(await ctx.mock.last(_gen)) + + def level(think: dict) -> str: + return str(think.get("thinkingLevel", "")).upper() + + _check( + ctx, + "custom-params", + "custom params of a workspace model reach the pipe (reasoning_effort=low " + "-> thinkingLevel LOW, include_thoughts=false; 2.5: thinking_budget=1024), " + "the chat's custom params win (reasoning_effort=high -> HIGH)", + (s1, s2) == (200, 200) + and r1.status == 200 + and level(think1) == "LOW" + and think1.get("includeThoughts") is False + and r2.status == 200 + and think2.get("thinkingBudget") == 1024 + and c.done + and level(think3) == "HIGH" + and think3.get("includeThoughts") is False, + f"models=[{s1},{s2}] model_params={think1} model_params_25={think2} " + f"chat_params={think3} chat_done={c.done} " + f"reasoning_effort_25={think4}", + ) + + +async def stream_chat(ctx: Ctx, b) -> None: + """Stream Chat Response off in the chat's Controls (honoured by the 0.12 + web UI): stream=false and params.stream_response=false.""" + await ctx.mock.reset() + c = await ctx.chat( + b, + PRO3, + directive([[TIMESTAMP], [CALC]]), + stream=False, + params={"stream_response": False}, + ) + reqs = await ctx.requests() + actions = [e.get("action") for e in reqs] + _check( + ctx, + "stream-chat", + "Stream Chat Response off for the chat (stream=false), two tool rounds: " + "the first request is not streamed, Open WebUI runs both tools, the turn " + "is saved as done with the final answer", + c.done + and not c.error + and actions + == ["generateContent", "streamGenerateContent", "streamGenerateContent"] + and [s for _, s, _ in _calls(c)] == ["completed", "completed"] + and "calculate_timestamp=" in _final(c), + _bdetail(c, reqs, f"actions={actions}"), + ) + + +async def stream_model(ctx: Ctx, b) -> None: + """Stream Chat Response off in the model's Advanced Params: Open WebUI sets + stream=false itself (the browser asks for a stream).""" + status, _ = await _upsert(ctx.t, NOSTREAM_MODEL, PRO3, {"stream_response": False}) + await ctx.mock.reset() + c = await ctx.chat(b, NOSTREAM_MODEL, directive([[TIMESTAMP]]), stream=True) + reqs = await ctx.requests() + actions = [e.get("action") for e in reqs] + _check( + ctx, + "stream-model", + "Stream Chat Response off for the model (Open WebUI sends stream=false), " + "a tool round: the call runs, the turn is saved as done", + status == 200 + and c.done + and not c.error + and actions == ["generateContent", "streamGenerateContent"] + and _calls(c) == [("get_current_timestamp", "completed", "mock-call-0-0")] + and _final(c).startswith("MOCK-FINAL get_current_timestamp="), + _bdetail(c, reqs, f"model={status} actions={actions}"), + ) diff --git a/tests/e2e/suites/gemini.py b/tests/e2e/suites/gemini.py index a5ab96c..f05199b 100644 --- a/tests/e2e/suites/gemini.py +++ b/tests/e2e/suites/gemini.py @@ -54,6 +54,11 @@ owuitools Open WebUI's built-in image generation / editing (image engine "gemini" on the mock) and code interpreter tools called by a Gemini text model + owui012 Open WebUI 0.12 features with native tool calling: context + compaction between tool rounds (and a failed or empty summary, + also before a reply), Tool Search, model controls and custom + params, Stream Chat Response off per chat and per model + (suites/_gemini_owui012.py; version-gated) The browser path sends the web UI's default params, i.e. native function calling with Open WebUI's built-in tools (functionDeclarations). The "images" @@ -103,6 +108,7 @@ "toolsapi", "tools", "owuitools", + "owui012", ) FID = "gemini" PATH = "pipelines/google/google_gemini.py" @@ -540,6 +546,7 @@ async def run(t: Suite) -> None: ("toolsapi", toolsapi_group), ("tools", tools_group), ("owuitools", owuitools_group), + ("owui012", owui012_group), ): if t.selected(group): try: @@ -2991,3 +2998,11 @@ async def owuitools_group(t: Suite, mock) -> None: from . import _gemini_owuitools await _gemini_owuitools.owuitools(t, mock) + + +# Open WebUI 0.12 features (compaction, Tool Search, model controls, streaming +# per chat): suites/_gemini_owui012.py +async def owui012_group(t: Suite, mock) -> None: + from . import _gemini_owui012 + + await _gemini_owui012.owui012(t, mock) From 4d421ad0c19cd401a085ff8fc0f689dca6d63e5e Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:52:10 +0200 Subject: [PATCH 05/12] docs(testing): gemini shared group, 562 / 551 checks, mutation round The gemini row and a new "Shared chats" section describe the shared group (13 checks on Open WebUI 0.12, shared.unsupported and shared.legacy before), the test-only redirect filter and the mutation round on the access logic. Counts per version (562 on v0.12.0-slim, 551 on v0.11.4-slim and v0.11.3-slim, all PASS in strict known mode), the timing of a full run and the meta-test (58 PASS / 389 FAIL / 0 KNOWN) as measured on 2026-10-11. The Gemini guide says that generated images are linked to their chat on every Open WebUI version and what an access check error refuses. Co-Authored-By: Claude Opus 5.5 --- docs/google-gemini-integration.md | 4 +- docs/testing.md | 119 +++++++++++++++++++++++------- 2 files changed, 95 insertions(+), 28 deletions(-) diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index 8683e8c..0f42a1e 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -430,7 +430,7 @@ Where the images of earlier turns come from: - **Saved chats** (the normal web UI chat): from the chat as Open WebUI stored it. That covers the generated images, which are attached to the answers as files (Open WebUI does not pass the files of earlier answers to a pipe), images you uploaded, images linked in earlier answers (pipeline versions before 1.15.2) and, when you regenerate an answer with guidance, the images of the user message whose answer you regenerate. Background tasks such as titles and follow-up suggestions get no images. - **API clients** (no saved chat): from the request's messages, as `image_url` parts (data URLs or Open WebUI file URLs) or markdown image links in the content, such as the `![Generated Image](/api/v1/files//content)` link the pipeline returns. - **Temporary chats**: Open WebUI does not save them, and the web UI sends only the images of your own messages. A generated image of an earlier turn is therefore not sent. To edit it, download it and attach it to your edit message. -- **Shared chats with "Allow replies"** (Open WebUI 0.12 and later): the people a chat or its folder is shared with in this mode write in the owner's chat, and their edit requests get the images of the whole chat like the owner's: the owner's uploads, the generated images of every answer and the images of the other participants. The pipeline links the images it generates to their chat, as Open WebUI does for its own image generation and for uploads, so the participants also see them. Images generated before pipeline 1.20.0 are not linked: Open WebUI shows them only to their owner and admins, and the pipeline does not send them for the other participants. A "Clone only" share or a read-only folder share lets people read the chat but not write in it: Open WebUI refuses their messages before the pipeline runs. +- **Shared chats with "Allow replies"** (Open WebUI 0.12 and later): the people a chat or its folder is shared with in this mode write in the owner's chat, and their edit requests get the images of the whole chat like the owner's: the owner's uploads, the generated images of every answer and the images of the other participants. Since 1.20.0 the pipeline links the images it generates to their chat (on every Open WebUI version), as Open WebUI does for its own image generation and for uploads, so Open WebUI also shows them to the participants and to the people a chat is shared with. Images generated before pipeline 1.20.0 are not linked: Open WebUI shows them only to their owner and admins, and the pipeline does not send them for the other participants. A "Clone only" share or a read-only folder share lets people read the chat but not write in it: Open WebUI refuses their messages before the pipeline runs. Which images are sent: @@ -439,7 +439,7 @@ Which images are sent: - Access follows Open WebUI: the pipeline never reads a chat or file that Open WebUI would not let the requesting user use, also when a message or an API request names the file id of another user. - Open WebUI 0.12 and later (detected by its access checks, not by the version): the history of a saved chat is read when Open WebUI lets the user continue the chat (`Chats.get_accessible_chat_by_id(..., permission="write")`, which Open WebUI itself checks before it calls the pipeline): the owner and the participants of a chat or folder shared with "Allow replies", users or groups. An admin is no exception: Open WebUI 0.12 gives admins read access to other users' chats, not write access. A file is read when it is the user's own, the user is an admin, or Open WebUI's `has_access_to_file` lets the user read it (for example through a shared chat or folder it is attached to, a knowledge base, a channel or a shared model), the same rule as Open WebUI's own file download. - Older Open WebUI versions: only the requesting user's own files and chats (an admin may read every file and chat). - - A file the user may not read is skipped and logged without its id. An error in an access check counts as a refusal: the chat's images then come from the request, as for API clients. + - A file the user may not read is skipped and logged without its id. An error in an access check counts as a refusal: such a file is skipped, and for the chat check the images of earlier turns come from the request, as for API clients. - Open WebUI 0.9.6 and later check the images they put into the request themselves in the same way (and also allow files shared with the user). Open WebUI 0.9.0 to 0.9.5 put the image of any file id named in a user message into the request before the pipeline runs, so update Open WebUI if users must not reach each other's files. - Earlier images are sent as parts of your message, which is how Gemini takes input images for editing. Earlier answers are not replayed as model turns, so no thought signatures are needed. diff --git a/docs/testing.md b/docs/testing.md index 2978d44..a131a04 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -68,10 +68,12 @@ total runtime: 1148s output: tests/e2e/out/20261009-111759-owui-e2e-111759-1234 ``` -A full run of all suites took about 19 minutes on a shared 8-CPU Docker host (1148 s on -v0.11.4-slim: 27 s container start-up, then gemini 343 s, azure 417 s, n8n 71 s, -infomaniak 36 s, filters 253 s; 1046 s on v0.12.0-slim, where azure takes 301 s because -the browser path's answers are saved as done sooner). Most of it is waiting: the azure +A full run of all suites took about 19 minutes on a shared 8-CPU Docker host, with +other e2e runs in parallel (2026-10-11, `google_gemini.py` 1.20.0: 1193 s on +v0.11.4-slim: 24 s container start-up, then gemini 370 s, azure 419 s, n8n 72 s, +infomaniak 36 s, filters 262 s; 1103 s on v0.12.0-slim, where azure takes 303 s because +the browser path's answers are saved as done sooner and gemini 400 s, about 35 s of it +the `shared` group, which only runs there). Most of it is waiting: the azure `rag` group adds about 5.5 minutes (query-generation timeouts, the query-generation pause and the 45 s retrieval limit are waited for; in the meta-test, where every mock answers HTTP 500, its requests fail at once), the gemini `tools`, `toolsapi`, `imgedit` and `owuitools` groups about three @@ -147,7 +149,7 @@ events), a **background title task** and that the server log has no unexpected | Suite | Function(s) | Mock | Groups (`--only .`) and file-specific scenarios | | --- | --- | --- | --- | -| `gemini` | `pipelines/google/google_gemini.py` (+ `google_search_tool`) | `mock_gemini.py`, `mock_tools.py` (tool servers, Jupyter) | `models` (image / video indicators, display names #172), `api` (thinking in the `
` block, full usage), `thinking` (summaries not replayed #176: the plain block of chats saved before 1.19.0 and the `type="reasoning"` blocks, also a stopped `done="false"` one; budget, level, include and strip valves; browser path: the live `done="false"` block through throttled `replace` events, no thinking status, the switch to `done="true"` with all thoughts on the first answer part and deltas after it, the duration up to the first answer part (mock trigger `paced-thinking`), Stop while thinking and the next turn without the block (`slow-thinking`), a tool round that clears its live block), `browser`, `tasks` (no `
` in task answers, no grounding tools for the tasks of a web_search chat), `image` (image models forced non-stream, exactly one saved file, the thinking block), `images` (thought images skipped, used as fallback, not used after IMAGE_SAFETY; dedup; two final images; image link for API clients; image history, where the limit keeps the current and the newest images and cuts more current images than it allows; optimization), `nano` (`gemini-nano-banana-2.1`), `imgvalve` (`IMAGE_GENERATION_MODELS`), `imgconfig` (ImageConfig valves, user valve, body), `imgtools` (tools per image model with web_search), `nostream` (`GOOGLE_STREAMING_ENABLED=false` with `stream=true`, #170), `video` (Veo: text and video saved, request shape, image-to-video), `grounding` (`google_search_tool` → googleSearch + urlContext, sources, `[1]` citations, Open WebUI's search statuses (the queries, then `Searched {{count}} sites` with `items` and no `urls`); no grounding without web_search), `vertex` (Vertex AI Search sources), `errors` (400, 500 with retry, blocked prompt, SAFETY finish, image error status, a streamed answer starting with `data:`), `retry` (`RETRY_COUNT` for streams), `status` (Stop leaves no running status), `valves` (model cache vs. valve changes, safety, whitelist, additional models, system prompt, user headers, API version, params, valve names and defaults), `concurrency` (forwarded user headers belong to the requesting user), `streamimg` (inline image in a stream), `imgedit` (image editing across turns, #194: in a saved chat the image of turn 1, attached as a file only, is sent with the edit request on `gemini-3.1-flash-image` and `gemini-nano-banana-2.1`; the follow-up task gets no images; uploads between generated turns in the default mode, with another image per turn (the mock's `final-image-` trigger) so their order shows; guided regeneration, also of an image-only message and of a text contained in the guidance; no guided regeneration after an empty answer, which Open WebUI 0.12 leaves out of the request (`imgedit.empty-answer`, 1.19.1); `IMAGE_HISTORY_MAX_REFERENCES` keeps the upload of the edit message, else the newest images; an image attached again is sent once and counts at its newest place (with `IMAGE_DEDUP_HISTORY=false` it is sent each time); older saved forms (a data: URL image file, markdown links to a file and to a data: URL; a text file is not sent); a temporary chat and a database error (injected by a test-only filter) take the history from the request; an image file of another user in a non-admin user's chat, in a markdown link on the API path and on the Veo path is not read (logged without its id), an admin continuing the user's chat reads it (up to Open WebUI 0.11; 0.12 gives an admin only read access to another user's chat and answers the admin's turn with HTTP 404 before the pipe runs, which `imgedit.admin` checks there instead)), `toolsapi` (native tool calling for API clients: client tools → `tool_calls` + `reasoning_details` streamed and non-streamed with `finish_reason` `tool_calls` (also as the openai SDK reads the stream), a text answer to a request with tools (`finish_reason` `stop`), `GOOGLE_STREAMING_ENABLED=false`, `MALFORMED_FUNCTION_CALL` / `UNEXPECTED_TOOL_CALL` streamed and non-streamed, continuation with and without signatures and with `reasoning_details` under `provider_specific_fields`, older turns with a thinking summary, empty arguments, content parts, a numeric id and a `function` that is not an object, `tool_choice` (also an undeclared name and with Search grounding on Gemini 3), name mapping (also a trailing newline) and duplicates, the `default_api.` prefix, schema clean-up, synthetic `owui_` ids, unchanged answers without tools), `tools` (native tool calling through Open WebUI's tool loop: built-in tool streamed and non-streamed, parallel calls (responses in call order), two rounds with summed usage, merged rounds without signatures, text before a call, thinking on / off, workspace Python tool, a tool result with an image, OpenAPI and MCP tool servers, direct tools of the browser, tool approval (approve, reject, two calls), follow-up turns on Gemini 3 / 2.5 / 3 and after an approved call, unknown tool, `MALFORMED_FUNCTION_CALL`, Search grounding with functions on Gemini 3 (the sources of both rounds, the next turn without web search) and grounding only on 2.5, title task, Legacy mode, `builtin_tools` off, `GOOGLE_STREAMING_ENABLED=false`; see [Native tool calling](#native-tool-calling-geminitools-geminitoolsapi)), `owuitools` (Open WebUI's built-in `generate_image`, `edit_image` and `execute_code` called by a Gemini text model: declared only with the chat toggles and for browser sessions, `edit_image` only with image editing on; the image engine `gemini` with `generateContent` and with `predict` (Imagen), editing an upload and the image of the previous turn, the image saved with the answer and its URL sent back to Gemini; a pipe image model makes its own image; the code interpreter with pyodide (the browser runs the code) and with Jupyter; see [Open WebUI's image and code tools](#open-webuis-image-and-code-tools-geminiowuitools)) | +| `gemini` | `pipelines/google/google_gemini.py` (+ `google_search_tool`) | `mock_gemini.py`, `mock_tools.py` (tool servers, Jupyter) | `models` (image / video indicators, display names #172), `api` (thinking in the `
` block, full usage), `thinking` (summaries not replayed #176: the plain block of chats saved before 1.19.0 and the `type="reasoning"` blocks, also a stopped `done="false"` one; budget, level, include and strip valves; browser path: the live `done="false"` block through throttled `replace` events, no thinking status, the switch to `done="true"` with all thoughts on the first answer part and deltas after it, the duration up to the first answer part (mock trigger `paced-thinking`), Stop while thinking and the next turn without the block (`slow-thinking`), a tool round that clears its live block), `browser`, `tasks` (no `
` in task answers, no grounding tools for the tasks of a web_search chat), `image` (image models forced non-stream, exactly one saved file, the thinking block), `images` (thought images skipped, used as fallback, not used after IMAGE_SAFETY; dedup; two final images; image link for API clients; image history, where the limit keeps the current and the newest images and cuts more current images than it allows; optimization), `nano` (`gemini-nano-banana-2.1`), `imgvalve` (`IMAGE_GENERATION_MODELS`), `imgconfig` (ImageConfig valves, user valve, body), `imgtools` (tools per image model with web_search), `nostream` (`GOOGLE_STREAMING_ENABLED=false` with `stream=true`, #170), `video` (Veo: text and video saved, request shape, image-to-video), `grounding` (`google_search_tool` → googleSearch + urlContext, sources, `[1]` citations, Open WebUI's search statuses (the queries, then `Searched {{count}} sites` with `items` and no `urls`); no grounding without web_search), `vertex` (Vertex AI Search sources), `errors` (400, 500 with retry, blocked prompt, SAFETY finish, image error status, a streamed answer starting with `data:`), `retry` (`RETRY_COUNT` for streams), `status` (Stop leaves no running status), `valves` (model cache vs. valve changes, safety, whitelist, additional models, system prompt, user headers, API version, params, valve names and defaults), `concurrency` (forwarded user headers belong to the requesting user), `streamimg` (inline image in a stream), `imgedit` (image editing across turns, #194: in a saved chat the image of turn 1, attached as a file only, is sent with the edit request on `gemini-3.1-flash-image` and `gemini-nano-banana-2.1`; the follow-up task gets no images; uploads between generated turns in the default mode, with another image per turn (the mock's `final-image-` trigger) so their order shows; guided regeneration, also of an image-only message and of a text contained in the guidance; no guided regeneration after an empty answer, which Open WebUI 0.12 leaves out of the request (`imgedit.empty-answer`, 1.19.1); `IMAGE_HISTORY_MAX_REFERENCES` keeps the upload of the edit message, else the newest images; an image attached again is sent once and counts at its newest place (with `IMAGE_DEDUP_HISTORY=false` it is sent each time); older saved forms (a data: URL image file, markdown links to a file and to a data: URL; a text file is not sent); a temporary chat and a database error (injected by a test-only filter) take the history from the request; an image file of another user in a non-admin user's chat, in a markdown link on the API path and on the Veo path is not read (logged without its id), an admin continuing the user's chat reads it (up to Open WebUI 0.11; 0.12 gives an admin only read access to another user's chat and answers the admin's turn with HTTP 404 before the pipe runs, which `imgedit.admin` checks there instead)), `shared` (chats shared with "Allow replies", Open WebUI 0.12, `google_gemini.py` 1.20.0; see [Shared chats](#shared-chats-geminishared)), `toolsapi` (native tool calling for API clients: client tools → `tool_calls` + `reasoning_details` streamed and non-streamed with `finish_reason` `tool_calls` (also as the openai SDK reads the stream), a text answer to a request with tools (`finish_reason` `stop`), `GOOGLE_STREAMING_ENABLED=false`, `MALFORMED_FUNCTION_CALL` / `UNEXPECTED_TOOL_CALL` streamed and non-streamed, continuation with and without signatures and with `reasoning_details` under `provider_specific_fields`, older turns with a thinking summary, empty arguments, content parts, a numeric id and a `function` that is not an object, `tool_choice` (also an undeclared name and with Search grounding on Gemini 3), name mapping (also a trailing newline) and duplicates, the `default_api.` prefix, schema clean-up, synthetic `owui_` ids, unchanged answers without tools), `tools` (native tool calling through Open WebUI's tool loop: built-in tool streamed and non-streamed, parallel calls (responses in call order), two rounds with summed usage, merged rounds without signatures, text before a call, thinking on / off, workspace Python tool, a tool result with an image, OpenAPI and MCP tool servers, direct tools of the browser, tool approval (approve, reject, two calls), follow-up turns on Gemini 3 / 2.5 / 3 and after an approved call, unknown tool, `MALFORMED_FUNCTION_CALL`, Search grounding with functions on Gemini 3 (the sources of both rounds, the next turn without web search) and grounding only on 2.5, title task, Legacy mode, `builtin_tools` off, `GOOGLE_STREAMING_ENABLED=false`; see [Native tool calling](#native-tool-calling-geminitools-geminitoolsapi)), `owuitools` (Open WebUI's built-in `generate_image`, `edit_image` and `execute_code` called by a Gemini text model: declared only with the chat toggles and for browser sessions, `edit_image` only with image editing on; the image engine `gemini` with `generateContent` and with `predict` (Imagen), editing an upload and the image of the previous turn, the image saved with the answer and its URL sent back to Gemini; a pipe image model makes its own image; the code interpreter with pyodide (the browser runs the code) and with Jupyter; see [Open WebUI's image and code tools](#open-webuis-image-and-code-tools-geminiowuitools)) | | `azure` | `pipelines/azure/azure_ai_foundry.py` | `mock_azure.py` (requires `api-version`; answers prompts with a `` block like a plain model, without `context`; query generation, embeddings, tool calls, a ~300 KB and a > 4 MiB stream event, `content: null`), `mock_search.py` (Azure AI Search and an App Service managed identity token endpoint, see below) | `valves` (from 3.0.0 also the `AZURE_AI_SEARCH_*` valves, their defaults, enums and the password input of the key, and no `AZURE_AI_SEARCH_MODE`), `models` (`AZURE_AI_MODEL` lists separated by `;`, `,` or spaces with exact names, `AZURE_AI_PIPELINE_PREFIX`, model from an `*.openai.azure.com` URL, predefined and fallback models), `api` (api-key and Bearer header, path and api-version, allow-listed body: extra client keys dropped, tools forwarded, `stream_options` only for streams; JSON 400 (exactly Azure's message, nothing appended) and text/plain 500 errors), `dotted` (`gpt-4.1`, `Phi-3.5-mini-instruct` reach upstream intact, model in header or body), `browser` (full status sequence, error status with exactly Azure's message), `tasks` (title task, also with Azure AI Search valves, #123), `rag` (the pipe's own Azure AI Search retrieval, 3.0.0, #187; `suites/_azure_rag.py`: API and browser path, stream and non-stream, Foundry `/models` endpoint; search request body per `query_type`, query text rules, embeddings for `deployment_name` (v1 route, gateway prefix, Bearer) and `endpoint` (api-version, own key or access token, chat key only on the same scheme, host and port), integrated vectorizer; `fields_mapping` with `select`, separator and fallback, list-valued titles; `filter`, `top_n_documents` (2 x candidates), `strictness` (per query), the api-version valve, the document budget (auto, also with a large `top_n_documents`, valve, unlimited, dropping a document); prompt placement, one block and one system message, `in_scope` and the tool-results clause, `role_information`, sanitizing (also nested tags, titles and file names), list and image-only content, Open WebUI's tool-images message, an attached file (`` and RAG template around the prompt; the search text is the typed prompt from `__metadata__`: `base_user_prompt` of Open WebUI 0.12 or `user_prompt` of 0.11, both also checked with the staged file in the driver process, `query-text.metadata`); citations, `context` event and `message.context`, scores per score type and with `AZURE_AI_INCLUDE_SEARCH_SCORES` off; `[docX]` → links, also split across stream deltas (API and browser, also without a finish chunk; in API streams all text before `[DONE]` and nothing after it), already linked or with parentheses in the URL (stream and non-stream), with a dotted deployment name; links in the history sent back as `[docX]` (also links saved before 2.8.0 with `)` in the URL; a hostile history line in linear time); only referenced sources saved and the show-all valve (also for an answer that cites only a document the search did not return, `[doc9]`, or one next to `[doc1]`); a context event over 128 KiB, an upstream event of ~300 KB (read and passed on) and one over 4 MiB (an error, API and browser); `content: null`; tool rounds (one search, reused only by a tool round of the same message with unchanged valves, no duplicate sources also when the tool round references a document, a non-stream tool call), tools and `stream_options` forwarded, tasks without retrieval and without `data_sources` (also for a saved chat without a websocket session); query generation (follow-ups, `always` / `off`, transcript with Open WebUI's conversation summary, max queries, `` with draft queries, fallback also for JSON nested too deep, partial and all-failed results, merge order by reciprocal rank fusion and reranker score, model in the body, the pause after 3 timeouts in a row: per model, reset by a success, still on after 16 s); auth: key, key valve, access token, system- and user-assigned managed identity (`mi_res_id`), token errors; fail-closed errors with a terminal status (HTTP 500/403/402/404/400/302/429/503, connection, retries, the 45 s retrieval limit, an answer larger than 16 MB, configuration errors, client `data_sources` (API stream and non-stream and the browser path as from an inlet filter: the error naming the removal of On Your Data in 3.0.0, the pipe's ERROR line and the terminal error status, nothing forwarded; an empty list is ignored), `context_length_exceeded` (exactly Azure's message and the budget hint)); Stop during a search and during query generation; `AZURE_AI_SEARCH_MODE=on_your_data` stored by a 2.9.0 pre-release or set in the environment is ignored; `log.debug`: the staged file run in the driver process with every logger at DEBUG logs no key or token, see below; `notice.none`: no On Your Data retirement notice of 2.8.1 in the server log), `logs` (no API key, no search key or token, no citation text, no generated queries, document titles or URLs in the server log at INFO and above) | | `n8n` | `pipelines/n8n/n8n.py` | `mock_n8n.py` | `api` (request payload contract, bearer / Cloudflare headers, usage, `intermediateSteps` tool display with verbosity and truncation, `` blocks, history / `INPUT_FIELD` / `RESPONSE_FIELD` valves, plain text, NDJSON, SSE streams in separate and coalesced writes with plain lines and `event:` / `id:` / `retry:` fields, OpenAI-style chunks, UTF-8 characters split across writes, braces inside strings, a large object trickling in as small writes (server CPU), webhook error), `browser` (saved answer, usage and final status for JSON, NDJSON, UTF-8, an n8n error chunk, a broken stream and a webhook error; chat context sent to the workflow for chat turns vs. background tasks; Stop during a stream and a non-stream request), `tasks` (title task without and with a chat id) | | `infomaniak` | `pipelines/infomaniak/infomaniak.py` | `mock_infomaniak.py` | `models` (llm models only, `NAME_PREFIX`), `api` (product id and bearer key, allow-listed body, SSE stream normal, coalesced into one write and split mid-JSON; OpenAI-style and Infomaniak `error.description` errors with one log line each), `browser` (saved answer, usage and status events for those streams plus no final newline, CRLF, a broken stream and an upstream error; Stop during the stream and while waiting for the response headers), `tasks` | @@ -180,7 +182,10 @@ every mapped host and is reissued when a name is missing) and starts `mocks/mock_la.py` (HTTPS on 127.0.0.1:443, control routes on :9105, managed identity endpoints on 127.0.0.1:9107 and 127.0.0.2:9107). The `ingest` group takes about two and a half minutes, mostly timeout, token-expiry and token back-off waits. It also creates the user -`filters-user@example.com`; the `gemini` suite creates `e2e-gemini-user@example.com`. +`filters-user@example.com`; the `gemini` suite creates `e2e-gemini-user@example.com`, +`e2e-imgedit-user@example.com` and, in its `shared` group, the non-admin users +`e2e-shared-*@example.com`, a folder of the owner and the group `E2E Shared Chats` +(deleted at the group's end). The `offline` group needs a fresh container (tiktoken keeps loaded encodings per process), so do not rerun it with `--reuse`. @@ -208,24 +213,26 @@ values among them), so a DEBUG server log cannot show what the pipe logs. Instea query generation, errors, client `data_sources` with a key, also streamed); no log record may contain a key or token. -Checks per suite in strict known mode, the same on Open WebUI **v0.12.0-slim**, -**v0.11.4-slim** and **v0.11.3-slim** (observed 2026-10-11 with the Open WebUI 0.12 -compatibility changes: `azure_ai_foundry.py` 3.0.1, `google_gemini.py` 1.19.1, Time -Token Tracker 2.7.0, `infomaniak.py` 2.2.2; 547 checks before them, plus -`azure.rag.query-text.metadata` and `gemini.imgedit.empty-answer`): - -| Suite | Checks | PASS / KNOWN | -| --- | ---: | ---: | -| `gemini` | 166 | 166 / 0 | -| `azure` | 199 | 199 / 0 | -| `n8n` | 51 | 51 / 0 | -| `infomaniak` | 32 | 32 / 0 | -| `filters` | 101 | 101 / 0 | -| **all** | **549** | **549 / 0** | - -No FAIL and no obsolete marker on any of the three versions. Two checks differ by -version: `gemini.imgedit.admin` expects on 0.12 that Open WebUI refuses an admin's turn in -another user's chat (HTTP 404), and on 0.11 that the pipe reads that chat. Before the +Checks per suite in strict known mode on Open WebUI **v0.12.0-slim**, **v0.11.4-slim** +and **v0.11.3-slim** (observed 2026-10-11 with `google_gemini.py` 1.20.0 and the +`gemini.shared` group, `azure_ai_foundry.py` 3.0.1, Time Token Tracker 2.7.0, +`infomaniak.py` 2.2.2; before the group 549 checks on every version, with +`google_gemini.py` 1.19.1): + +| Suite | Checks on v0.12.0 | Checks on v0.11.4 / v0.11.3 | PASS / KNOWN | +| --- | ---: | ---: | ---: | +| `gemini` | 179 | 168 | all / 0 | +| `azure` | 199 | 199 | all / 0 | +| `n8n` | 51 | 51 | all / 0 | +| `infomaniak` | 32 | 32 | all / 0 | +| `filters` | 101 | 101 | all / 0 | +| **all** | **562** | **551** | **all / 0** | + +No FAIL and no obsolete marker on any of the three versions. The `gemini.shared` group +has 13 checks on 0.12 and two before (`shared.unsupported`, `shared.legacy`), see +[Shared chats](#shared-chats-geminishared). `gemini.imgedit.admin` expects on 0.12 that +Open WebUI refuses an admin's turn in another user's chat (HTTP 404), and on 0.11 that +the pipe reads that chat. Before the compatibility changes, `main` (31a4e7d) had 542 PASS and 5 FAIL on v0.12.0-slim: the `server-log` checks of azure, n8n and infomaniak (the module names), `imgedit.admin` and `azure.rag.query-text.attached`. The checks of the 1.19.0 thinking block @@ -394,6 +401,65 @@ the PNG). Details start with `mock_ok=` as in the tool groups; the extra tokens 10 checks FAIL (`generate`, `generate-predict`, `edit`, `edit-followup`, `code`, `code-jupyter`). +### Shared chats (`gemini.shared`) + +The group lives in `tests/e2e/suites/_gemini_shared.py` (run by `suites/gemini.py` +after `imgedit`). Open WebUI 0.12 shares a chat with users or groups (`POST +/api/v1/chats//share` with `share_mode: "continue"` for "Allow replies", then `POST +/api/v1/chats/shared//access/update` with the grants) or a folder (`POST +/api/v1/folders//access/update`). The group drives these APIs with five non-admin +users: the owner A, a participant B (user grant on chat X, "Allow replies"), a member G +of a group with a grant on X, a viewer D (chat Y shared "Clone only" with D, and Y in a +folder of A shared read-only with D) and an outsider C. Chat X starts with A's turn on +`gemini-3.1-flash-image` with the upload `upload1` (generated image `final`); B, A and +G then write in it (the mock's `final-image-` trigger gives each turn its own image). + +A test-only global filter (`e2e_chat_redirect`, installed for the group) hands the pipe +another chat when an API request's prompt carries `e2e-chat=/`: it sets `chat_id` and `user_message_id` of `__metadata__`, as if Open WebUI had +called the pipe for that chat without its own check. So the pipe's own check is tested +also for the users Open WebUI turns away before the pipe. `e2e-chat-fault=` +and `e2e-file-fault=` make the pipe's next call of +`Chats.get_accessible_chat_by_id` / `has_access_to_file` for that id raise (an inert, +signature-preserving async wrapper stays installed in the server process, so the pipe's +feature detection still sees the helpers on a later load). + +| Check | What it shows | +| --- | --- | +| `shared.visible` | Open WebUI lets B read A's upload and the generated image of X (`GET /api/v1/files//content`): the pipe links its images to the chat | +| `shared.participant` | B's browser turn in X sends `upload1` and `final` (the edit request as in `imgedit`) | +| `shared.owner` | A's next turn also sends B's generated image (`gen2`, a file of B) | +| `shared.group` | G's turn sends every image of X | +| `shared.refused` | D's turn in Y and C's turn in X: Open WebUI answers HTTP 404, no request reaches the mock | +| `shared.pipe-participant` | filter: B is handed X, the images are sent (shows that the filter works) | +| `shared.pipe-admin` | filter: the admin (read access to every chat in 0.12, no write access) gets none of X | +| `shared.pipe-viewer` | filter: D (read access to Y through the Clone only share and the read-only folder) gets none of Y | +| `shared.pipe-outsider` | filter: C gets none of X | +| `shared.chat-fault` | the chat check raises for B: nothing of X is sent, the error is logged, the answer completes | +| `shared.links` | B's API request with markdown links: A's upload of X is read; a file of the admin and a file of A outside X are not (logged twice, without their ids) | +| `shared.file-fault` | the file check raises for B's link to A's upload: it is not read, logged without its id | +| `shared.video` | Veo image-to-video: B's markdown link to A's upload of X is sent as `instances[0].image` | + +On Open WebUI before 0.12 the group records `shared.unsupported` (PASS, `feature not in +Open WebUI 0.11.4`) and `shared.legacy`: A's chat shared with B through a 0.11 share +link with a user grant, Open WebUI lets B read the upload and the generated image, but the +pipe keeps its owner / admin rule and does not read B's markdown link to the upload. +With `google_gemini.py` 1.19.1 on v0.12.0-slim, 10 of the 13 checks FAIL; `refused`, +`pipe-viewer` and `pipe-outsider` pass (1.19.1 allowed only the owner or an admin). +The group takes about 35 s on 0.12 and a few seconds on 0.11. + +A mutation round on the access logic (2026-10-11, `--only 'gemini.(shared|imgedit)$'`) +killed every mutant: the chat check always allowing (`pipe-admin`, `pipe-viewer`: the +file check still refuses the outsider), `permission="read"` (`pipe-admin`, +`pipe-viewer`), chat check errors allowing (`chat-fault`), the file check allowing +after `has_access_to_file` (`links` and `imgedit.foreign`, `.api-foreign`, +`.video-foreign`), file check errors allowing (`file-fault`), the feature detection +always off (9 `shared` checks), an admin bypass of the chat check (`pipe-admin`), no +linking of generated images (`visible`, `participant`, `owner`, `group`, +`pipe-participant`; on 0.11.4 `legacy`); on v0.11.4-slim the feature detection always on +(13 `imgedit` checks and `legacy`) and `has_access_to_file` used without the 0.12 +detection (`legacy`). The last two cannot differ on 0.12. + ### API path vs. browser path - **API path** (`OWUI.chat`): `POST /api/chat/completions` without `chat_id`, like an @@ -580,7 +646,8 @@ harness/ driver library: owui.py (REST client, API-path chat, valves suites/ one module per suite: GROUPS + async def run(t: Suite); _*.py are helper modules of a suite, not suites (_azure_rag.py: the azure rag group; _gemini_tools.py: gemini.tools / toolsapi; - _gemini_owuitools.py: gemini.owuitools) + _gemini_owuitools.py: gemini.owuitools; + _gemini_shared.py: gemini.shared) mocks/ aiohttp provider mocks + serve_all.py (127.0.0.1:9101-9104 and :9106 in the container), tool servers mock_tools.py (OpenAPI and Jupyter :9111, MCP :9112, no E2E_MOCK_FAULT); mock_la.py (Log Analytics) is started @@ -820,7 +887,7 @@ demand (*Actions → E2E → Run workflow*, with an image tag and a suites input | Job | What it does | | --- | --- | | `e2e` | `run.sh` with all suites in **strict known mode** (`E2E_STRICT_KNOWN=1`) against the default image, which is read from the `DEFAULT_IMAGE=` line of `run.sh` (the only place it is defined). The weekly run adds `ghcr.io/open-webui/open-webui:latest-slim`; a manual run uses the image tag input. Output directory as artifact, `summary.md` as job summary | -| `meta` | **Meta-test**: `main`'s function files (`--ref origin/main`) with every provider mock answering HTTP 500 (`E2E_MOCK_FAULT=500`), suites `gemini azure n8n infomaniak`. Nearly everything fails, and it must give **no KNOWN**: a KNOWN means the `evidence` of that known bug also matches an unrelated failure and would hide it. `REQUEST_SIDE_KNOWN` in the workflow may list bugs that can only show in the request the pipe sends upstream (they reproduce whatever the mock answers); it is empty, because the evidence of every registered bug also needs proof that the upstream answered (e.g. `HTTP 200` and the mock's answer). Observed 2026-10-09 (`main` 53b8495): 38 PASS / 197 FAIL / 0 KNOWN; on 2026-10-10 with `main` 3ff6cf9 (Azure pipeline 2.8.1) and the harness of the Azure pipeline 3.0.0 (azure `rag` group): 35 PASS / 327 FAIL / 0 KNOWN (466 s), and with `main` 4b80e24 (Azure pipeline 3.0.0, `google_gemini.py` 1.17.0) and the suites with the `gemini.tools` / `toolsapi` / `imgedit` groups: 57 PASS / 370 FAIL / 0 KNOWN (557 s); with `main` 2fc7bcc (`google_gemini.py` 1.18.0, `infomaniak.py` 2.2.2, no known bug registered) and the suites with the `owuitools` group and the checks of 1.19.0: 57 PASS / 386 FAIL / 0 KNOWN (545 s); on 2026-10-11 with `main` 31a4e7d (`google_gemini.py` 1.19.0, Azure pipeline 3.0.0) and the harness of the Open WebUI 0.12 changes (2 more checks): 57 PASS / 388 FAIL / 0 KNOWN (518 s). The `filters` suite is left out because it uses no provider mock (its known bugs reproduce for real) | +| `meta` | **Meta-test**: `main`'s function files (`--ref origin/main`) with every provider mock answering HTTP 500 (`E2E_MOCK_FAULT=500`), suites `gemini azure n8n infomaniak`. Nearly everything fails, and it must give **no KNOWN**: a KNOWN means the `evidence` of that known bug also matches an unrelated failure and would hide it. `REQUEST_SIDE_KNOWN` in the workflow may list bugs that can only show in the request the pipe sends upstream (they reproduce whatever the mock answers); it is empty, because the evidence of every registered bug also needs proof that the upstream answered (e.g. `HTTP 200` and the mock's answer). Observed 2026-10-09 (`main` 53b8495): 38 PASS / 197 FAIL / 0 KNOWN; on 2026-10-10 with `main` 3ff6cf9 (Azure pipeline 2.8.1) and the harness of the Azure pipeline 3.0.0 (azure `rag` group): 35 PASS / 327 FAIL / 0 KNOWN (466 s), and with `main` 4b80e24 (Azure pipeline 3.0.0, `google_gemini.py` 1.17.0) and the suites with the `gemini.tools` / `toolsapi` / `imgedit` groups: 57 PASS / 370 FAIL / 0 KNOWN (557 s); with `main` 2fc7bcc (`google_gemini.py` 1.18.0, `infomaniak.py` 2.2.2, no known bug registered) and the suites with the `owuitools` group and the checks of 1.19.0: 57 PASS / 386 FAIL / 0 KNOWN (545 s); on 2026-10-11 with `main` 31a4e7d (`google_gemini.py` 1.19.0, Azure pipeline 3.0.0) and the harness of the Open WebUI 0.12 changes (2 more checks): 57 PASS / 388 FAIL / 0 KNOWN (518 s); with `main` 6b99dcf (`google_gemini.py` 1.19.1) and the harness with the `gemini.shared` group (2 more checks on the default image): 58 PASS / 389 FAIL / 0 KNOWN (536 s). The `filters` suite is left out because it uses no provider mock (its known bugs reproduce for real) | | `api` | `check_owui_api.sh latest` | `E2E_TIMEOUT` and the steps' `timeout-minutes` bound every job, so a hanging scenario From afd8a69b5f40e36a9ed40ab2dff24e07400d5738 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:53:16 +0200 Subject: [PATCH 06/12] docs: Gemini 1.20.0 with Open WebUI 0.12 Tool Search, compaction and controls google-gemini-integration.md: Tool Search (deferred tools found by search_tools are declared from 1.20.0), context compaction during tool runs (what Open WebUI sends after a compaction, the summary request and the empty answer for a failed or empty summary, the reordering when the boundary falls into an earlier turn), stream=false tool rounds on 0.12, and the per-chat thinking parameters (reasoning_effort, thinking_budget, include_thoughts) with the note that 0.12 model controls are refused for pipe models. README: feature bullets. testing.md: the gemini.owui012 group (checks, version gating, the mock's compaction answer and triggers, the 1.19.1 failures and the no-stream experiment), counts (gemini 178, all 561) and timing: all suites in strict known mode 561 PASS / 0 FAIL / 0 KNOWN on v0.12.0-slim (1105 s), v0.11.4-slim (1215 s) and v0.11.3-slim (1232 s); meta-test with main 6b99dcf 58 PASS / 399 FAIL / 0 KNOWN (527 s); smoke dry-run 15 PASS, 2 SKIP, 38 requests (76 s). +semver: none Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- docs/google-gemini-integration.md | 36 ++++++++++-- docs/testing.md | 98 +++++++++++++++++++++++-------- 3 files changed, 106 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index b4428ee..c5b73ef 100644 --- a/README.md +++ b/README.md @@ -236,11 +236,12 @@ The functions include a built-in encryption mechanism for sensitive information: - **Nano Banana image models**: `gemini-nano-banana-2.1`, `gemini-3.1-flash-image` and `gemini-3.1-flash-lite-image` are detected as image models; each generated image is uploaded once (interim thought images are skipped unless no final image arrives). API clients get generated images and videos as links in the answer. Google Search grounding is left out for the image models without Search support (`gemini-2.5-flash-image`, `gemini-3.1-flash-lite-image`). Newer image models outside Gemini 3 / Nano Banana naming can be added via `GOOGLE_IMAGE_GENERATION_MODELS` without a code change. - **Video generation (Veo)**: Generate videos with Google Veo models (3.1, 3, 2). Configurable aspect ratio, resolution, duration, negative prompt, and person generation controls. Supports text-to-video and image-to-video for all supported Veo models. Videos are automatically uploaded and embedded with playback controls. - **Token usage tracking**: Returns prompt, completion, and total token counts to Open WebUI for automatic persistence in the database, for streaming and non-streaming responses. -- **Background tasks**: Works as the Open WebUI task model for title, tag and follow-up generation (without Google Search grounding, even when the chat uses it). +- **Background tasks**: Works as the Open WebUI task model for title, tag and follow-up generation (without Google Search grounding, even when the chat uses it) and for context compaction, where a failed or empty summary leaves Open WebUI's own summary of the compacted messages instead of an error text. - **Model whitelist & additional models**: Restrict the visible model list via `GOOGLE_MODEL_WHITELIST` and add SDK-unsupported models via `GOOGLE_MODEL_ADDITIONAL`; changes show on the next model list refresh, without waiting for the model cache TTL. - Grounding with Google Search via the [google_search_tool.py filter](./filters/google_search_tool.py) - Grounding with Vertex AI Search via the [vertex_ai_search_tool.py filter](./filters/vertex_ai_search_tool.py) - [Native tool calling through Open WebUI's tool loop](./docs/google-gemini-integration.md#native-tool-calling) (built-in, workspace, MCP, OpenAPI, terminal and direct tools, tool approval); tool calls are returned to API clients +- Works with Open WebUI 0.12's [Tool Search](./docs/google-gemini-integration.md#tool-search-open-webui-012) (tools found by `search_tools` are declared to Gemini so it can call them) and [compaction during long tool runs](./docs/google-gemini-integration.md#context-compaction-during-tool-runs) - Configurable API version support 🔗 [Google Gemini Pipeline in Open WebUI](https://openwebui.com/f/owndev/google_gemini) diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index 3c828ee..6a59e6e 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -95,7 +95,7 @@ For a pip or uv installation of Open WebUI, run `pip install "google-genai>=1.68 Forward user information headers (like Name, Id, Email and Role) to Google API or LiteLLM for better context and analytics. The headers always belong to the user of the request, also with concurrent requests of several users. Also, change the base URL for the Google Generative AI API if needed. - **Native tool calling** - Tools offered by Open WebUI (built-in, workspace, MCP, OpenAPI, terminal and direct tools) are declared to Gemini, and Open WebUI runs Gemini's tool calls, including tool approval. API clients get the calls back as OpenAI `tool_calls`. See [Native tool calling](#native-tool-calling). + Tools offered by Open WebUI (built-in, workspace, MCP, OpenAPI, terminal and direct tools) are declared to Gemini, and Open WebUI runs Gemini's tool calls, including tool approval. API clients get the calls back as OpenAI `tool_calls`. Works with Open WebUI 0.12's Tool Search and with context compaction between tool rounds. See [Native tool calling](#native-tool-calling). ## Environment Variables @@ -759,7 +759,7 @@ Gemini 3 requires the thought signatures of its function calls back in the follo - In a turn with tool calls, Gemini's thinking appears as Open WebUI's own "Thought for N seconds" block. The duration can read 0 seconds, because the thoughts of a round arrive together with its tool calls. With `GOOGLE_INCLUDE_THOUGHTS=false` Gemini 3 still sends thought signatures, so the block appears without content for every tool round. - Answers without tool calls show the thinking as a `
` block in the answer, which Open WebUI renders as the same localized block (see [Thinking Summaries in Conversation History](#thinking-summaries-in-conversation-history)). - The answer after a tool call is streamed as it arrives. It has no inline `[1]` citation markers; its Google Search sources are still attached. -- For a chat request with `stream=false`, the pipeline answers a round with tool calls as a stream, because Open WebUI runs tools only for streamed answers. With `GOOGLE_STREAMING_ENABLED=false` the pipeline still calls Gemini without streaming. +- For a chat request with `stream=false` (**Stream Chat Response** off in the chat's Controls, which Open WebUI 0.12 honours per chat, or in the model's Advanced Params), the pipeline answers a round with tool calls as a stream, because Open WebUI runs tools only for streamed answers. Open WebUI 0.12 saves a non-streamed reply with tool calls as finished, but still does not run its tools (0.11 left it generating). With `GOOGLE_STREAMING_ENABLED=false` the pipeline still calls Gemini without streaming. ### API clients @@ -780,9 +780,23 @@ Requests to `/api/chat/completions` without a chat pass the client's `tools` to Gemini accepts function names of up to 64 characters (letters, digits, `_` and `-`, starting with a letter or `_`). Other tool names, for example an OpenAPI operation `lookup.v2`, are declared with the invalid characters replaced and a short hash appended (`lookup_v2_`), and Gemini's calls are mapped back to the original name. Parameter schemas are sent as JSON Schema (`parameters_json_schema`) after a light clean-up: `$schema`, `$id`, `$comment`, `examples`, `deprecated`, `readOnly`, `writeOnly` and `x-*` keys and boolean `exclusiveMinimum` / `exclusiveMaximum` are removed, tuple `items` become `prefixItems`, and `required` only lists existing properties. Tools without parameters are declared without a schema. +### Tool Search (Open WebUI 0.12) + +With **Tool Search** on (experimental, **Admin Settings → Interface**, `ENABLE_TOOL_SEARCH`), Open WebUI leaves tools whose definition is longer than the **Deferral Threshold** (`TOOL_SEARCH_DEFER_THRESHOLD`, characters) out of the request, unless they match **Always Loaded Tools** (`TOOL_SEARCH_ALWAYS_LOADED`); with **Defer Built-in Tools** (`TOOL_SEARCH_DEFER_BUILTIN_TOOLS`, default on) its built-in tools count too. It lists them by name and a short description in an `` block of the system prompt and adds its `search_tools` tool, which returns the full definitions of the tools that match a query. The model is then expected to call a found tool by name. + +Gemini only calls functions that are declared in the request. From version 1.20.0 the pipeline therefore also declares every deferred tool that a `search_tools` result in the conversation names, with the definition Open WebUI runs. A tool found in one round can be called in the next round, and in later turns of the chat while its search result is in the history. Without this (1.19.1 and older) Gemini could not call the found tool. Names Gemini does not accept are mapped as for any other tool (see [Names and schemas](#names-and-schemas)). + +### Context compaction during tool runs + +With context compaction on (**Admin Settings → Interface**, `ENABLE_CONTEXT_COMPACTION`) Open WebUI replaces older messages with a summary once the chat passes the token threshold. Open WebUI 0.12 also does this between the rounds of a long tool run: the next request then holds the system prompt with a `[CONVERSATION SUMMARY]` block, the current user message and the most recent tool rounds. Open WebUI keeps every tool call together with its results and keeps their `reasoning_details`, so the pipeline sends Gemini 3 each remaining call with its thought signature. Rounds that were compacted away are only in the summary. + +The summary itself comes from the compaction model, by default the chat's model, i.e. this pipeline. That request carries no tools and gets no thinking block. When it fails (for example HTTP 429 or 500) or Gemini returns no text (for example because its thinking used up the task's `max_tokens`, 1000 unless the task model parameters set another value), the pipeline answers it with an empty text from version 1.20.0, and Open WebUI uses its own summary of the compacted messages. Up to 1.19.1 the pipeline's error text or `[No content generated]` became the conversation summary; when Open WebUI compacts before a reply, it saves that summary with the chat, so later turns saw the error text in place of the earlier messages. + +Open WebUI may move the current user message before tool rounds of an earlier turn when the compaction boundary falls into that turn. The pipeline still sends a valid request; the order is Open WebUI's. + ### Requests without tools -Image generation models, background tasks (title, tag and follow-up generation) and chats with **Function Calling** set to **Legacy** declare no functions. Legacy mode remains available: Open WebUI then selects and runs the tools itself before it calls the pipeline. +Image generation models, background tasks (title, tag and follow-up generation, context compaction) and chats with **Function Calling** set to **Legacy** declare no functions. Legacy mode remains available: Open WebUI then selects and runs the tools itself before it calls the pipeline. ### Open WebUI's image generation and code interpreter @@ -817,7 +831,7 @@ Gemini then calls `generate_image` with a prompt, or `edit_image` with a prompt - Open WebUI ends the tool loop after `CHAT_RESPONSE_MAX_TOOL_CALL_ITERATIONS` rounds (default 256). The pipeline adds no limit of its own. - Tool approval on Open WebUI 0.11.4: after an approved call the saved message loses the tool result, and when Gemini calls several tools at once, only the first call is asked for and run. -- Tool calling was verified with Open WebUI 0.11.3 and 0.11.4. To check it against the Gemini API with your own key, for example with a new Gemini model, run the real-API smoke test in a clone of this repository (see the [testing guide](./testing.md#real-api-smoke-test-manual-needs-a-key)). +- Tool calling was verified with Open WebUI 0.11.3, 0.11.4 and 0.12.0. To check it against the Gemini API with your own key, for example with a new Gemini model, run the real-API smoke test in a clone of this repository (see the [testing guide](./testing.md#real-api-smoke-test-manual-needs-a-key)). ## Default System Prompt @@ -987,6 +1001,20 @@ response = client.models.generate_content( print(response.text) ``` +### Setting the per-chat values in Open WebUI + +The pipeline reads three request parameters: + +| Parameter | Models | Effect | +| --- | --- | --- | +| `reasoning_effort` | Gemini 3 and Nano Banana | thinking level, checked against the model's levels as above; overrides `GOOGLE_THINKING_LEVEL` | +| `thinking_budget` | Gemini 2.5 and other models without thinking levels | overrides `GOOGLE_THINKING_BUDGET` | +| `include_thoughts` | all thinking models | `false` hides the thoughts; `GOOGLE_INCLUDE_THOUGHTS=false` always wins | + +Set them as **Advanced Params** of a model (**Workspace → Models** or **Admin Settings → Models**: **Reasoning Effort** has its own field, the others go under **Custom Parameters**) or in a chat's **Controls**, whose values win over the model's. `reasoning_effort` does not change the thinking budget of Gemini 2.5 models. + +The **model controls** of Open WebUI 0.12 (named controls in the model editor whose options set custom parameters) cannot be used for this pipeline: Open WebUI refuses them for pipe models and for workspace models based on one ("Model controls require a server-managed provider model."). + ## Token Usage Tracking The pipeline automatically extracts token usage metadata from every Gemini response and returns it to Open WebUI so it can be saved to the database and displayed in the UI. diff --git a/docs/testing.md b/docs/testing.md index 2978d44..ed3ee49 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -50,32 +50,35 @@ Typical output (abridged): starting owui-e2e-111759-1234 from ghcr.io/open-webui/open-webui:v0.11.4-slim Open WebUI healthy after 20s (http://localhost:57765) Open WebUI 0.11.4, suites: gemini, azure, n8n, infomaniak, filters -function versions: google_gemini.py 1.19.1, azure_ai_foundry.py 3.0.1, n8n.py 2.3.1, ... +function versions: google_gemini.py 1.20.0, azure_ai_foundry.py 3.0.1, n8n.py 2.3.1, ... === gemini === [PASS ] gemini.load pipelines/google/google_gemini.py loads (create, import, activate) [PASS ] gemini.api.stream API stream without websocket session: answer streamed, thinking in the
block ... [PASS ] gemini.tools.builtin built-in tool, browser stream=True: Open WebUI runs it, ... ... ---- gemini: 343s +--- gemini: 398s ... === infomaniak === ... [PASS ] infomaniak.models.name-prefix NAME_PREFIX valve changes the model names, and changing it back restores them ... -SUMMARY: 549 PASS, 0 FAIL, 0 KNOWN in 1121s -total runtime: 1148s +SUMMARY: 561 PASS, 0 FAIL, 0 KNOWN in 1179s +total runtime: 1215s output: tests/e2e/out/20261009-111759-owui-e2e-111759-1234 ``` -A full run of all suites took about 19 minutes on a shared 8-CPU Docker host (1148 s on -v0.11.4-slim: 27 s container start-up, then gemini 343 s, azure 417 s, n8n 71 s, -infomaniak 36 s, filters 253 s; 1046 s on v0.12.0-slim, where azure takes 301 s because -the browser path's answers are saved as done sooner). Most of it is waiting: the azure -`rag` group adds about 5.5 minutes (query-generation timeouts, the query-generation -pause and the 45 s retrieval limit are waited for; in the meta-test, where every mock answers HTTP 500, its requests fail at -once), the gemini `tools`, `toolsapi`, `imgedit` and `owuitools` groups about three -minutes and the filters `ingest` group (Logs Ingestion API) another 2.5 minutes. The +A full run of all suites took about 20 minutes on a shared 8-CPU Docker host (1215 s on +v0.11.4-slim: 23 s container start-up, then gemini 398 s, azure 420 s, n8n 72 s, +infomaniak 36 s, filters 251 s; 1105 s on v0.12.0-slim, where azure takes 303 s because +the browser path's answers are saved as done sooner; 1232 s on v0.11.3-slim; observed +2026-10-11 with `google_gemini.py` 1.20.0, two runs side by side). Most of it is +waiting: the azure `rag` group adds about 5.5 minutes (query-generation timeouts, the +query-generation pause and the 45 s retrieval limit are waited for; in the meta-test, +where every mock answers HTTP 500, its requests fail at once), the gemini `tools`, +`toolsapi`, `imgedit` and `owuitools` groups about three minutes, the gemini `owui012` +group about one minute and the filters `ingest` group (Logs Ingestion API) another 2.5 +minutes. The browser scenarios of the gemini `thinking` group (paced streams for the live block and its duration, Stop while thinking, a tool round) take about 20 s. Network downloads on first use come on top @@ -147,7 +150,7 @@ events), a **background title task** and that the server log has no unexpected | Suite | Function(s) | Mock | Groups (`--only .`) and file-specific scenarios | | --- | --- | --- | --- | -| `gemini` | `pipelines/google/google_gemini.py` (+ `google_search_tool`) | `mock_gemini.py`, `mock_tools.py` (tool servers, Jupyter) | `models` (image / video indicators, display names #172), `api` (thinking in the `
` block, full usage), `thinking` (summaries not replayed #176: the plain block of chats saved before 1.19.0 and the `type="reasoning"` blocks, also a stopped `done="false"` one; budget, level, include and strip valves; browser path: the live `done="false"` block through throttled `replace` events, no thinking status, the switch to `done="true"` with all thoughts on the first answer part and deltas after it, the duration up to the first answer part (mock trigger `paced-thinking`), Stop while thinking and the next turn without the block (`slow-thinking`), a tool round that clears its live block), `browser`, `tasks` (no `
` in task answers, no grounding tools for the tasks of a web_search chat), `image` (image models forced non-stream, exactly one saved file, the thinking block), `images` (thought images skipped, used as fallback, not used after IMAGE_SAFETY; dedup; two final images; image link for API clients; image history, where the limit keeps the current and the newest images and cuts more current images than it allows; optimization), `nano` (`gemini-nano-banana-2.1`), `imgvalve` (`IMAGE_GENERATION_MODELS`), `imgconfig` (ImageConfig valves, user valve, body), `imgtools` (tools per image model with web_search), `nostream` (`GOOGLE_STREAMING_ENABLED=false` with `stream=true`, #170), `video` (Veo: text and video saved, request shape, image-to-video), `grounding` (`google_search_tool` → googleSearch + urlContext, sources, `[1]` citations, Open WebUI's search statuses (the queries, then `Searched {{count}} sites` with `items` and no `urls`); no grounding without web_search), `vertex` (Vertex AI Search sources), `errors` (400, 500 with retry, blocked prompt, SAFETY finish, image error status, a streamed answer starting with `data:`), `retry` (`RETRY_COUNT` for streams), `status` (Stop leaves no running status), `valves` (model cache vs. valve changes, safety, whitelist, additional models, system prompt, user headers, API version, params, valve names and defaults), `concurrency` (forwarded user headers belong to the requesting user), `streamimg` (inline image in a stream), `imgedit` (image editing across turns, #194: in a saved chat the image of turn 1, attached as a file only, is sent with the edit request on `gemini-3.1-flash-image` and `gemini-nano-banana-2.1`; the follow-up task gets no images; uploads between generated turns in the default mode, with another image per turn (the mock's `final-image-` trigger) so their order shows; guided regeneration, also of an image-only message and of a text contained in the guidance; no guided regeneration after an empty answer, which Open WebUI 0.12 leaves out of the request (`imgedit.empty-answer`, 1.19.1); `IMAGE_HISTORY_MAX_REFERENCES` keeps the upload of the edit message, else the newest images; an image attached again is sent once and counts at its newest place (with `IMAGE_DEDUP_HISTORY=false` it is sent each time); older saved forms (a data: URL image file, markdown links to a file and to a data: URL; a text file is not sent); a temporary chat and a database error (injected by a test-only filter) take the history from the request; an image file of another user in a non-admin user's chat, in a markdown link on the API path and on the Veo path is not read (logged without its id), an admin continuing the user's chat reads it (up to Open WebUI 0.11; 0.12 gives an admin only read access to another user's chat and answers the admin's turn with HTTP 404 before the pipe runs, which `imgedit.admin` checks there instead)), `toolsapi` (native tool calling for API clients: client tools → `tool_calls` + `reasoning_details` streamed and non-streamed with `finish_reason` `tool_calls` (also as the openai SDK reads the stream), a text answer to a request with tools (`finish_reason` `stop`), `GOOGLE_STREAMING_ENABLED=false`, `MALFORMED_FUNCTION_CALL` / `UNEXPECTED_TOOL_CALL` streamed and non-streamed, continuation with and without signatures and with `reasoning_details` under `provider_specific_fields`, older turns with a thinking summary, empty arguments, content parts, a numeric id and a `function` that is not an object, `tool_choice` (also an undeclared name and with Search grounding on Gemini 3), name mapping (also a trailing newline) and duplicates, the `default_api.` prefix, schema clean-up, synthetic `owui_` ids, unchanged answers without tools), `tools` (native tool calling through Open WebUI's tool loop: built-in tool streamed and non-streamed, parallel calls (responses in call order), two rounds with summed usage, merged rounds without signatures, text before a call, thinking on / off, workspace Python tool, a tool result with an image, OpenAPI and MCP tool servers, direct tools of the browser, tool approval (approve, reject, two calls), follow-up turns on Gemini 3 / 2.5 / 3 and after an approved call, unknown tool, `MALFORMED_FUNCTION_CALL`, Search grounding with functions on Gemini 3 (the sources of both rounds, the next turn without web search) and grounding only on 2.5, title task, Legacy mode, `builtin_tools` off, `GOOGLE_STREAMING_ENABLED=false`; see [Native tool calling](#native-tool-calling-geminitools-geminitoolsapi)), `owuitools` (Open WebUI's built-in `generate_image`, `edit_image` and `execute_code` called by a Gemini text model: declared only with the chat toggles and for browser sessions, `edit_image` only with image editing on; the image engine `gemini` with `generateContent` and with `predict` (Imagen), editing an upload and the image of the previous turn, the image saved with the answer and its URL sent back to Gemini; a pipe image model makes its own image; the code interpreter with pyodide (the browser runs the code) and with Jupyter; see [Open WebUI's image and code tools](#open-webuis-image-and-code-tools-geminiowuitools)) | +| `gemini` | `pipelines/google/google_gemini.py` (+ `google_search_tool`) | `mock_gemini.py`, `mock_tools.py` (tool servers, Jupyter) | `models` (image / video indicators, display names #172), `api` (thinking in the `
` block, full usage), `thinking` (summaries not replayed #176: the plain block of chats saved before 1.19.0 and the `type="reasoning"` blocks, also a stopped `done="false"` one; budget, level, include and strip valves; browser path: the live `done="false"` block through throttled `replace` events, no thinking status, the switch to `done="true"` with all thoughts on the first answer part and deltas after it, the duration up to the first answer part (mock trigger `paced-thinking`), Stop while thinking and the next turn without the block (`slow-thinking`), a tool round that clears its live block), `browser`, `tasks` (no `
` in task answers, no grounding tools for the tasks of a web_search chat), `image` (image models forced non-stream, exactly one saved file, the thinking block), `images` (thought images skipped, used as fallback, not used after IMAGE_SAFETY; dedup; two final images; image link for API clients; image history, where the limit keeps the current and the newest images and cuts more current images than it allows; optimization), `nano` (`gemini-nano-banana-2.1`), `imgvalve` (`IMAGE_GENERATION_MODELS`), `imgconfig` (ImageConfig valves, user valve, body), `imgtools` (tools per image model with web_search), `nostream` (`GOOGLE_STREAMING_ENABLED=false` with `stream=true`, #170), `video` (Veo: text and video saved, request shape, image-to-video), `grounding` (`google_search_tool` → googleSearch + urlContext, sources, `[1]` citations, Open WebUI's search statuses (the queries, then `Searched {{count}} sites` with `items` and no `urls`); no grounding without web_search), `vertex` (Vertex AI Search sources), `errors` (400, 500 with retry, blocked prompt, SAFETY finish, image error status, a streamed answer starting with `data:`), `retry` (`RETRY_COUNT` for streams), `status` (Stop leaves no running status), `valves` (model cache vs. valve changes, safety, whitelist, additional models, system prompt, user headers, API version, params, valve names and defaults), `concurrency` (forwarded user headers belong to the requesting user), `streamimg` (inline image in a stream), `imgedit` (image editing across turns, #194: in a saved chat the image of turn 1, attached as a file only, is sent with the edit request on `gemini-3.1-flash-image` and `gemini-nano-banana-2.1`; the follow-up task gets no images; uploads between generated turns in the default mode, with another image per turn (the mock's `final-image-` trigger) so their order shows; guided regeneration, also of an image-only message and of a text contained in the guidance; no guided regeneration after an empty answer, which Open WebUI 0.12 leaves out of the request (`imgedit.empty-answer`, 1.19.1); `IMAGE_HISTORY_MAX_REFERENCES` keeps the upload of the edit message, else the newest images; an image attached again is sent once and counts at its newest place (with `IMAGE_DEDUP_HISTORY=false` it is sent each time); older saved forms (a data: URL image file, markdown links to a file and to a data: URL; a text file is not sent); a temporary chat and a database error (injected by a test-only filter) take the history from the request; an image file of another user in a non-admin user's chat, in a markdown link on the API path and on the Veo path is not read (logged without its id), an admin continuing the user's chat reads it (up to Open WebUI 0.11; 0.12 gives an admin only read access to another user's chat and answers the admin's turn with HTTP 404 before the pipe runs, which `imgedit.admin` checks there instead)), `toolsapi` (native tool calling for API clients: client tools → `tool_calls` + `reasoning_details` streamed and non-streamed with `finish_reason` `tool_calls` (also as the openai SDK reads the stream), a text answer to a request with tools (`finish_reason` `stop`), `GOOGLE_STREAMING_ENABLED=false`, `MALFORMED_FUNCTION_CALL` / `UNEXPECTED_TOOL_CALL` streamed and non-streamed, continuation with and without signatures and with `reasoning_details` under `provider_specific_fields`, older turns with a thinking summary, empty arguments, content parts, a numeric id and a `function` that is not an object, `tool_choice` (also an undeclared name and with Search grounding on Gemini 3), name mapping (also a trailing newline) and duplicates, the `default_api.` prefix, schema clean-up, synthetic `owui_` ids, unchanged answers without tools), `tools` (native tool calling through Open WebUI's tool loop: built-in tool streamed and non-streamed, parallel calls (responses in call order), two rounds with summed usage, merged rounds without signatures, text before a call, thinking on / off, workspace Python tool, a tool result with an image, OpenAPI and MCP tool servers, direct tools of the browser, tool approval (approve, reject, two calls), follow-up turns on Gemini 3 / 2.5 / 3 and after an approved call, unknown tool, `MALFORMED_FUNCTION_CALL`, Search grounding with functions on Gemini 3 (the sources of both rounds, the next turn without web search) and grounding only on 2.5, title task, Legacy mode, `builtin_tools` off, `GOOGLE_STREAMING_ENABLED=false`; see [Native tool calling](#native-tool-calling-geminitools-geminitoolsapi)), `owuitools` (Open WebUI's built-in `generate_image`, `edit_image` and `execute_code` called by a Gemini text model: declared only with the chat toggles and for browser sessions, `edit_image` only with image editing on; the image engine `gemini` with `generateContent` and with `predict` (Imagen), editing an upload and the image of the previous turn, the image saved with the answer and its URL sent back to Gemini; a pipe image model makes its own image; the code interpreter with pyodide (the browser runs the code) and with Jupyter; see [Open WebUI's image and code tools](#open-webuis-image-and-code-tools-geminiowuitools)), `owui012` (Open WebUI 0.12 features with native tool calling, version-gated: on 0.11 a missing feature passes with "feature not in Open WebUI "; context compaction between tool rounds (the summary request without tools; afterwards the summary, the user message and the kept rounds with their signatures; a follow-up turn whose boundary falls into the earlier turn), a failed or empty summary request between tool rounds and before a reply (also the summary saved with the chat, also on 0.11) leaves Open WebUI's own summary (1.20.0), Tool Search (`search_tools` and the `` list, the found tool declared and run in the next round and in the next turn, a mapped OpenAPI name; 1.20.0), model controls refused for pipe models and the custom params a control would set, Stream Chat Response off per chat and per model in a tool turn; see [Open WebUI 0.12 features](#open-webui-012-features-geminiowui012)) | | `azure` | `pipelines/azure/azure_ai_foundry.py` | `mock_azure.py` (requires `api-version`; answers prompts with a `` block like a plain model, without `context`; query generation, embeddings, tool calls, a ~300 KB and a > 4 MiB stream event, `content: null`), `mock_search.py` (Azure AI Search and an App Service managed identity token endpoint, see below) | `valves` (from 3.0.0 also the `AZURE_AI_SEARCH_*` valves, their defaults, enums and the password input of the key, and no `AZURE_AI_SEARCH_MODE`), `models` (`AZURE_AI_MODEL` lists separated by `;`, `,` or spaces with exact names, `AZURE_AI_PIPELINE_PREFIX`, model from an `*.openai.azure.com` URL, predefined and fallback models), `api` (api-key and Bearer header, path and api-version, allow-listed body: extra client keys dropped, tools forwarded, `stream_options` only for streams; JSON 400 (exactly Azure's message, nothing appended) and text/plain 500 errors), `dotted` (`gpt-4.1`, `Phi-3.5-mini-instruct` reach upstream intact, model in header or body), `browser` (full status sequence, error status with exactly Azure's message), `tasks` (title task, also with Azure AI Search valves, #123), `rag` (the pipe's own Azure AI Search retrieval, 3.0.0, #187; `suites/_azure_rag.py`: API and browser path, stream and non-stream, Foundry `/models` endpoint; search request body per `query_type`, query text rules, embeddings for `deployment_name` (v1 route, gateway prefix, Bearer) and `endpoint` (api-version, own key or access token, chat key only on the same scheme, host and port), integrated vectorizer; `fields_mapping` with `select`, separator and fallback, list-valued titles; `filter`, `top_n_documents` (2 x candidates), `strictness` (per query), the api-version valve, the document budget (auto, also with a large `top_n_documents`, valve, unlimited, dropping a document); prompt placement, one block and one system message, `in_scope` and the tool-results clause, `role_information`, sanitizing (also nested tags, titles and file names), list and image-only content, Open WebUI's tool-images message, an attached file (`` and RAG template around the prompt; the search text is the typed prompt from `__metadata__`: `base_user_prompt` of Open WebUI 0.12 or `user_prompt` of 0.11, both also checked with the staged file in the driver process, `query-text.metadata`); citations, `context` event and `message.context`, scores per score type and with `AZURE_AI_INCLUDE_SEARCH_SCORES` off; `[docX]` → links, also split across stream deltas (API and browser, also without a finish chunk; in API streams all text before `[DONE]` and nothing after it), already linked or with parentheses in the URL (stream and non-stream), with a dotted deployment name; links in the history sent back as `[docX]` (also links saved before 2.8.0 with `)` in the URL; a hostile history line in linear time); only referenced sources saved and the show-all valve (also for an answer that cites only a document the search did not return, `[doc9]`, or one next to `[doc1]`); a context event over 128 KiB, an upstream event of ~300 KB (read and passed on) and one over 4 MiB (an error, API and browser); `content: null`; tool rounds (one search, reused only by a tool round of the same message with unchanged valves, no duplicate sources also when the tool round references a document, a non-stream tool call), tools and `stream_options` forwarded, tasks without retrieval and without `data_sources` (also for a saved chat without a websocket session); query generation (follow-ups, `always` / `off`, transcript with Open WebUI's conversation summary, max queries, `` with draft queries, fallback also for JSON nested too deep, partial and all-failed results, merge order by reciprocal rank fusion and reranker score, model in the body, the pause after 3 timeouts in a row: per model, reset by a success, still on after 16 s); auth: key, key valve, access token, system- and user-assigned managed identity (`mi_res_id`), token errors; fail-closed errors with a terminal status (HTTP 500/403/402/404/400/302/429/503, connection, retries, the 45 s retrieval limit, an answer larger than 16 MB, configuration errors, client `data_sources` (API stream and non-stream and the browser path as from an inlet filter: the error naming the removal of On Your Data in 3.0.0, the pipe's ERROR line and the terminal error status, nothing forwarded; an empty list is ignored), `context_length_exceeded` (exactly Azure's message and the budget hint)); Stop during a search and during query generation; `AZURE_AI_SEARCH_MODE=on_your_data` stored by a 2.9.0 pre-release or set in the environment is ignored; `log.debug`: the staged file run in the driver process with every logger at DEBUG logs no key or token, see below; `notice.none`: no On Your Data retirement notice of 2.8.1 in the server log), `logs` (no API key, no search key or token, no citation text, no generated queries, document titles or URLs in the server log at INFO and above) | | `n8n` | `pipelines/n8n/n8n.py` | `mock_n8n.py` | `api` (request payload contract, bearer / Cloudflare headers, usage, `intermediateSteps` tool display with verbosity and truncation, `` blocks, history / `INPUT_FIELD` / `RESPONSE_FIELD` valves, plain text, NDJSON, SSE streams in separate and coalesced writes with plain lines and `event:` / `id:` / `retry:` fields, OpenAI-style chunks, UTF-8 characters split across writes, braces inside strings, a large object trickling in as small writes (server CPU), webhook error), `browser` (saved answer, usage and final status for JSON, NDJSON, UTF-8, an n8n error chunk, a broken stream and a webhook error; chat context sent to the workflow for chat turns vs. background tasks; Stop during a stream and a non-stream request), `tasks` (title task without and with a chat id) | | `infomaniak` | `pipelines/infomaniak/infomaniak.py` | `mock_infomaniak.py` | `models` (llm models only, `NAME_PREFIX`), `api` (product id and bearer key, allow-listed body, SSE stream normal, coalesced into one write and split mid-JSON; OpenAI-style and Infomaniak `error.description` errors with one log line each), `browser` (saved answer, usage and status events for those streams plus no final newline, CRLF, a broken stream and an upstream error; Stop during the stream and while waiting for the response headers), `tasks` | @@ -209,23 +212,26 @@ query generation, errors, client `data_sources` with a key, also streamed); no l contain a key or token. Checks per suite in strict known mode, the same on Open WebUI **v0.12.0-slim**, -**v0.11.4-slim** and **v0.11.3-slim** (observed 2026-10-11 with the Open WebUI 0.12 -compatibility changes: `azure_ai_foundry.py` 3.0.1, `google_gemini.py` 1.19.1, Time -Token Tracker 2.7.0, `infomaniak.py` 2.2.2; 547 checks before them, plus -`azure.rag.query-text.metadata` and `gemini.imgedit.empty-answer`): +**v0.11.4-slim** and **v0.11.3-slim** (observed 2026-10-11 with `google_gemini.py` +1.20.0, `azure_ai_foundry.py` 3.0.1, Time Token Tracker 2.7.0, `infomaniak.py` 2.2.2: +549 checks before the 12 of the `gemini.owui012` group; 547 before the Open WebUI 0.12 +compatibility changes, which added `azure.rag.query-text.metadata` and +`gemini.imgedit.empty-answer`): | Suite | Checks | PASS / KNOWN | | --- | ---: | ---: | -| `gemini` | 166 | 166 / 0 | +| `gemini` | 178 | 178 / 0 | | `azure` | 199 | 199 / 0 | | `n8n` | 51 | 51 / 0 | | `infomaniak` | 32 | 32 / 0 | | `filters` | 101 | 101 / 0 | -| **all** | **549** | **549 / 0** | +| **all** | **561** | **561 / 0** | -No FAIL and no obsolete marker on any of the three versions. Two checks differ by +No FAIL and no obsolete marker on any of the three versions. Some checks differ by version: `gemini.imgedit.admin` expects on 0.12 that Open WebUI refuses an admin's turn in -another user's chat (HTTP 404), and on 0.11 that the pipe reads that chat. Before the +another user's chat (HTTP 404), and on 0.11 that the pipe reads that chat; on 0.11 the +`gemini.owui012` checks of 0.12 features check what 0.11 does instead (see [Open WebUI +0.12 features](#open-webui-012-features-geminiowui012)). Before the compatibility changes, `main` (31a4e7d) had 542 PASS and 5 FAIL on v0.12.0-slim: the `server-log` checks of azure, n8n and infomaniak (the module names), `imgedit.admin` and `azure.rag.query-text.attached`. The checks of the 1.19.0 thinking block @@ -291,7 +297,7 @@ content of the current turn without the issued signature or `fc_mode`, `allowed_names`, `include_flag`, `kinds` (role and part kinds per content), `fc` / `fr` (calls with id, name, args and signature state `none` / `skip` / `issued` / `bad`; responses with id, name and response), `round`, `directive`, `answer` (`fc`, -`final`, `text`, `malformed`, `mock-error`, `http-`), `issued`, +`final`, `text`, `summary`, `malformed`, `mock-error`, `http-`), `issued`, `server_echoed`, `synthetic_ids_upstream` and `orphan_fr`. **Tool mocks.** `mocks/mock_tools.py`, started by `serve_all.py`, is an OpenAPI tool @@ -394,6 +400,45 @@ the PNG). Details start with `mock_ok=` as in the tool groups; the extra tokens 10 checks FAIL (`generate`, `generate-predict`, `edit`, `edit-followup`, `code`, `code-jupyter`). +### Open WebUI 0.12 features (`gemini.owui012`) + +The group lives in `tests/e2e/suites/_gemini_owui012.py` (run by `suites/gemini.py` +after `owuitools`). Through `/api/v1/chats/config` it switches on context compaction +(threshold and cap 1 token) for the compaction scenarios and Tool Search (deferral +threshold 0, `get_current_timestamp` always loaded, built-in tools deferred) for the +Tool Search scenarios; it creates the workspace models `e2e-g012-*`, and at its end it +restores the chat settings and the tool server connections and deletes the models. +A check of a 0.12 feature passes on 0.11 with "feature not in Open WebUI " +in its title and checks what that version does instead (no compaction between tool +rounds, no `search_tools`, `model_controls` stored as an ordinary parameter). Details +start with `mock_ok=` (the preflight of the tool groups) and `owui=`. + +| Check | What it shows | +| --- | --- | +| `owui012.compaction` | three tool rounds on `gemini-3-pro-preview`: 0.12 compacts before the third and the fourth request; both summary requests reach the pipe without tools (`summaries=['summary', 'summary']`); the next request has `[CONVERSATION SUMMARY]` and the mock's summary in the system instruction, the user message and the last round only (`kinds=[user:text,model:fc,user:fr]`, the call with its issued signature, no orphan response); all rounds stay saved | +| `owui012.compaction-followup` | a tool turn after a turn with two rounds: the boundary falls into the earlier turn and Open WebUI puts the current user message before that turn's kept round and answer (`prompt_first=True`, `kinds=`); the request is still valid (every call with its response, first calls signed) | +| `owui012.summary-error`, `owui012.summary-empty` | the summary request between tool rounds fails (mock trigger `summary-400`) or has no text (`summary-empty`: the thought part only, `MAX_TOKENS`): the next request carries Open WebUI's fallback summary (`- user: ...`), not the pipe's error text or `[No content generated]` | +| `owui012.saved-summary` | compaction before the third reply (0.11 and 0.12), the summary request fails: the `contextSummary` Open WebUI saves on the second user message and the summary it sends are its fallback, not the error text | +| `owui012.toolsearch` | `search_tools` and `get_current_timestamp` declared, the deferred `calculate_timestamp` only in ``; after `search_tools` returned it, it is declared in the next request, called and run | +| `owui012.toolsearch-followup` | the next turn calls the found tool directly: it is declared from the search result in the history | +| `owui012.toolsearch-mapped` | the deferred OpenAPI tool `lookup.v2`: declared as `lookup_v2_` after the search, the tool server gets the call | +| `owui012.controls` | 0.12 refuses model controls on a workspace model of a pipe model (HTTP 400 "Model controls require a server-managed provider model.") | +| `owui012.custom-params` | custom params of a workspace model (`reasoning_effort=low`, `include_thoughts=false`; `thinking_budget=1024` on 2.5) reach the pipe and the chat's custom params win (`reasoning_effort=high`); `reasoning_effort_25=` records, without asserting it, that `reasoning_effort` leaves a 2.5 budget unchanged | +| `owui012.stream-chat` | Stream Chat Response off for the chat (`stream=false`, `params.stream_response=false`), two tool rounds: the first request is not streamed, both tools run, the turn is done | +| `owui012.stream-model` | Stream Chat Response off in the model's Advanced Params: Open WebUI sends `stream=false`; the tool round runs, the turn is done | + +The Gemini mock answers Open WebUI's compaction prompt ("Summarize the conversation +history that will be compacted ...") with `MOCK-SUMMARY of `, the mock's call +ids in its "Messages Being Compacted" section (`answer=summary` in the record); the +triggers `summary-400` and `summary-empty` in the user message reach only that request, +because the prompt contains the chat's messages. With `google_gemini.py` 1.19.1 six +checks FAIL on 0.12.0: `summary-error`, `summary-empty`, `saved-summary`, `toolsearch`, +`toolsearch-followup` and `toolsearch-mapped`; on 0.11.4 `saved-summary` FAILs too +(`stored=["Error generating content: 400 INVALID_ARGUMENT. ..."]`). A copy of 1.20.0 that answers a +`stream=false` tool round of a chat with a `chat.completion` instead of a stream fails +`stream-chat` and `stream-model` on 0.12.0 (the turn is done, but the tool never runs) +and on 0.11.4 (the turn never ends): the pipe's stream for such rounds is still needed. + ### API path vs. browser path - **API path** (`OWUI.chat`): `POST /api/chat/completions` without `chat_id`, like an @@ -580,7 +625,8 @@ harness/ driver library: owui.py (REST client, API-path chat, valves suites/ one module per suite: GROUPS + async def run(t: Suite); _*.py are helper modules of a suite, not suites (_azure_rag.py: the azure rag group; _gemini_tools.py: gemini.tools / toolsapi; - _gemini_owuitools.py: gemini.owuitools) + _gemini_owuitools.py: gemini.owuitools; + _gemini_owui012.py: gemini.owui012) mocks/ aiohttp provider mocks + serve_all.py (127.0.0.1:9101-9104 and :9106 in the container), tool servers mock_tools.py (OpenAPI and Jupyter :9111, MCP :9112, no E2E_MOCK_FAULT); mock_la.py (Log Analytics) is started @@ -801,9 +847,9 @@ runner and the script both took 10 values from the file and found none in the ou **Dry run.** `--dry-run-mock` checks the runner itself without a key: every scenario runs end to end against the e2e Gemini mock, which answers the `MOCKTOOLS:` directive instead of following the prompt. Observed 2026-10-09 on v0.11.4-slim (and 2026-10-10 with -`google_gemini.py` 1.19.0): 15 PASS (R1-R8, R10, R12-R15, +`google_gemini.py` 1.19.0, 2026-10-11 with 1.20.0): 15 PASS (R1-R8, R10, R12-R15, `LOG`, `SECRETS`), 0 FAIL, 2 SKIP (R9 without Vertex credentials, R11 note), 38 -generate requests (the mock received the same 38), 90-160 s including the container +generate requests (the mock received the same 38), 76-160 s including the container start. The mock always follows the directive, so no retry happens. Its answers prove only the mechanics: real results can differ. With `--max-requests 5` the proxy forwarded the two preflight requests and three of R1, answered R1's fourth with HTTP 429 (the mock @@ -820,7 +866,7 @@ demand (*Actions → E2E → Run workflow*, with an image tag and a suites input | Job | What it does | | --- | --- | | `e2e` | `run.sh` with all suites in **strict known mode** (`E2E_STRICT_KNOWN=1`) against the default image, which is read from the `DEFAULT_IMAGE=` line of `run.sh` (the only place it is defined). The weekly run adds `ghcr.io/open-webui/open-webui:latest-slim`; a manual run uses the image tag input. Output directory as artifact, `summary.md` as job summary | -| `meta` | **Meta-test**: `main`'s function files (`--ref origin/main`) with every provider mock answering HTTP 500 (`E2E_MOCK_FAULT=500`), suites `gemini azure n8n infomaniak`. Nearly everything fails, and it must give **no KNOWN**: a KNOWN means the `evidence` of that known bug also matches an unrelated failure and would hide it. `REQUEST_SIDE_KNOWN` in the workflow may list bugs that can only show in the request the pipe sends upstream (they reproduce whatever the mock answers); it is empty, because the evidence of every registered bug also needs proof that the upstream answered (e.g. `HTTP 200` and the mock's answer). Observed 2026-10-09 (`main` 53b8495): 38 PASS / 197 FAIL / 0 KNOWN; on 2026-10-10 with `main` 3ff6cf9 (Azure pipeline 2.8.1) and the harness of the Azure pipeline 3.0.0 (azure `rag` group): 35 PASS / 327 FAIL / 0 KNOWN (466 s), and with `main` 4b80e24 (Azure pipeline 3.0.0, `google_gemini.py` 1.17.0) and the suites with the `gemini.tools` / `toolsapi` / `imgedit` groups: 57 PASS / 370 FAIL / 0 KNOWN (557 s); with `main` 2fc7bcc (`google_gemini.py` 1.18.0, `infomaniak.py` 2.2.2, no known bug registered) and the suites with the `owuitools` group and the checks of 1.19.0: 57 PASS / 386 FAIL / 0 KNOWN (545 s); on 2026-10-11 with `main` 31a4e7d (`google_gemini.py` 1.19.0, Azure pipeline 3.0.0) and the harness of the Open WebUI 0.12 changes (2 more checks): 57 PASS / 388 FAIL / 0 KNOWN (518 s). The `filters` suite is left out because it uses no provider mock (its known bugs reproduce for real) | +| `meta` | **Meta-test**: `main`'s function files (`--ref origin/main`) with every provider mock answering HTTP 500 (`E2E_MOCK_FAULT=500`), suites `gemini azure n8n infomaniak`. Nearly everything fails, and it must give **no KNOWN**: a KNOWN means the `evidence` of that known bug also matches an unrelated failure and would hide it. `REQUEST_SIDE_KNOWN` in the workflow may list bugs that can only show in the request the pipe sends upstream (they reproduce whatever the mock answers); it is empty, because the evidence of every registered bug also needs proof that the upstream answered (e.g. `HTTP 200` and the mock's answer). Observed 2026-10-09 (`main` 53b8495): 38 PASS / 197 FAIL / 0 KNOWN; on 2026-10-10 with `main` 3ff6cf9 (Azure pipeline 2.8.1) and the harness of the Azure pipeline 3.0.0 (azure `rag` group): 35 PASS / 327 FAIL / 0 KNOWN (466 s), and with `main` 4b80e24 (Azure pipeline 3.0.0, `google_gemini.py` 1.17.0) and the suites with the `gemini.tools` / `toolsapi` / `imgedit` groups: 57 PASS / 370 FAIL / 0 KNOWN (557 s); with `main` 2fc7bcc (`google_gemini.py` 1.18.0, `infomaniak.py` 2.2.2, no known bug registered) and the suites with the `owuitools` group and the checks of 1.19.0: 57 PASS / 386 FAIL / 0 KNOWN (545 s); on 2026-10-11 with `main` 31a4e7d (`google_gemini.py` 1.19.0, Azure pipeline 3.0.0) and the harness of the Open WebUI 0.12 changes (2 more checks): 57 PASS / 388 FAIL / 0 KNOWN (518 s); with `main` 6b99dcf (`google_gemini.py` 1.19.1) and the harness with the `gemini.owui012` group (12 more checks): 58 PASS / 399 FAIL / 0 KNOWN (527 s; `owui012.controls` passes, it needs no provider answer). The `filters` suite is left out because it uses no provider mock (its known bugs reproduce for real) | | `api` | `check_owui_api.sh latest` | `E2E_TIMEOUT` and the steps' `timeout-minutes` bound every job, so a hanging scenario From 7d29afdaa56d8d8f9d96a9d3d15cc82266a11886 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:57:20 +0200 Subject: [PATCH 07/12] docs: valve types are public API too (Open WebUI 0.12 validates stored valves) Since Open WebUI 0.12, saving a function validates its stored valves against the new Valves model (models/functions.py, utils/valves.py). A changed valve type can therefore block the update for users who set that valve. +semver: none Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index d942cc4..4704b0a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -76,7 +76,7 @@ Expose a `Filter` class with `inlet(body)` and/or `outlet(body)`, mutating the r ## Invariants to preserve -- **Valve names are the public API.** Never rename or remove one; add new valves with backward-compatible defaults. +- **Valve names and types are the public API.** Never rename or remove one; add new valves with backward-compatible defaults. Do not change a valve's type either: since Open WebUI 0.12, saving a function validates its stored valves against the new `Valves` model, so a type change can block the update for users who set that valve. - **Secrets:** assigning to an `EncryptedStr` field encrypts (stored with an `encrypted:` prefix). Call `EncryptedStr.decrypt(...)` only at the point of use — never store the decrypted value on `self`, never log it. - **Body allow-list:** `pipe()` filters the incoming body through an explicit `allowed_params` set before forwarding upstream. Add new provider params to that set deliberately; do not forward `body` wholesale. - **Status events:** emit via `__event_emitter__` with `{"type": "status"|"chat:*", "data": {...}}` at start, at streaming start, and on completion *or* error. Never leave a request without a terminal status. From 34436712f6a03f370bef772c7968aa44574cad45 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:04:24 +0200 Subject: [PATCH 08/12] fix(gemini): thinking levels per Gemini 3 model; invalid thinking params no longer drop the thinking config - Gemini 3 text models now get the thinking_level values Google lists per model (https://ai.google.dev/gemini-api/docs/thinking): gemini-3-flash, 3.5-flash-lite and 3.6-flash "minimal".."high", gemini-3.1-pro and 3.8-flash "low"/"medium"/"high", gemini-3-pro "low"/"high". Before, every Gemini 3 text model was limited to "low"/"high", so "medium" became "high" and "minimal" became "low". Unknown Gemini 3 text models keep "low"/"high". - A reasoning_effort that is not a string, or a thinking_budget that is not a number, is ignored with a warning. Before, it raised inside the thinking setup and silently dropped the whole thinking config (include_thoughts too). A number sent as a string ("1024") is accepted. New checks gemini.thinking.level-models and gemini.thinking.params-robust; both fail with the previous pipeline. +semver: none Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/google-gemini-integration.md | 20 +++++++---- pipelines/google/google_gemini.py | 57 +++++++++++++++++++++++++++---- tests/e2e/suites/gemini.py | 56 ++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 2f3ec38..47cd337 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ The functions include a built-in encryption mechanism for sensitive information: - Sends messages from Open WebUI to **Google Gemini**. - Supports encryption of sensitive information such as API keys. - Supports both streaming and non-streaming responses (streaming is automatically disabled for image generation models, including released IDs such as `gemini-3.1-flash-image`). -- **Thinking & reasoning**: Configurable thinking levels (`low` / `high`; `minimal` / `high` for `gemini-3.1-flash-image` and `gemini-3.1-flash-lite-image`, `minimal` / `medium` / `high` for `gemini-nano-banana-2.1`) for Gemini 3 models and thinking budgets (0–32 768 tokens) for Gemini 2.5 models, with per-chat override support. +- **Thinking & reasoning**: Configurable thinking levels for Gemini 3 models, with the levels each model supports (for example `low` / `high` on `gemini-3-pro`, `minimal` to `high` on `gemini-3-flash`, `low` / `medium` / `high` on `gemini-3.1-pro`, `minimal` / `high` on `gemini-3.1-flash-image` and `gemini-3.1-flash-lite-image`, `minimal` / `medium` / `high` on `gemini-nano-banana-2.1`) and thinking budgets (0–32 768 tokens) for Gemini 2.5 models, with per-chat override support. - **Native reasoning display**: Thinking appears as Open WebUI's own reasoning block, live ("Thinking..." with a spinner) while Gemini thinks and as "Thought for N seconds" afterwards, with the title in each user's Open WebUI language. Google Search grounding uses Open WebUI's localized search statuses (the queries, then "Searched N sites"). - **Lean history**: Previously rendered thinking summaries are stripped from assistant messages before the conversation is replayed to the API (configurable). - Provides configurable error handling and timeouts: temporary API errors are retried (`GOOGLE_RETRY_COUNT`) for streaming and non-streaming requests, and every started status (image, video) is closed when a request fails or is stopped. diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index 3cc2f74..20e15f7 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -59,10 +59,10 @@ For a pip or uv installation of Open WebUI, run `pip install "google-genai>=1.68 > Streaming is automatically disabled for image generation models to prevent chunk size issues. Image models are recognized by their preview and released IDs (for example `gemini-3.1-flash-image-preview`, `gemini-3.1-flash-image`, `gemini-3.1-flash-lite-image`) and by Nano Banana IDs such as `gemini-nano-banana-2.1`; Imagen models (`imagen-*`) are not Gemini image models. Newer image models can be added without a code change via `GOOGLE_IMAGE_GENERATION_MODELS` (see [Additional image generation models](#additional-image-generation-models)). If a model that is not recognized still returns an image while streaming, the image is uploaded and attached once the stream ends. - **Thinking Support** - Support reasoning and thinking steps, allowing models to break down complex tasks. Includes configurable thinking levels for Gemini 3 Pro ("low"/"high") and thinking budgets (0-32768 tokens) for other thinking-capable models. The thoughts are shown as Open WebUI's own reasoning block, titled in each user's language (see [Thinking Summaries in Conversation History](#thinking-summaries-in-conversation-history)). + Support reasoning and thinking steps, allowing models to break down complex tasks. Includes configurable thinking levels for Gemini 3 models (the levels each model supports, see [Thinking Levels](#thinking-levels-gemini-3-models)) and thinking budgets (0-32768 tokens) for other thinking-capable models. The thoughts are shown as Open WebUI's own reasoning block, titled in each user's language (see [Thinking Summaries in Conversation History](#thinking-summaries-in-conversation-history)). > [!Note] - > **Thinking Levels vs Thinking Budgets**: Gemini 3 Pro models use `thinking_level` ("low" or "high"), while other models like Gemini 2.5 use `thinking_budget` (token count). See [Gemini Thinking Documentation](https://ai.google.dev/gemini-api/docs/thinking) for details. + > **Thinking Levels vs Thinking Budgets**: Gemini 3 models use `thinking_level` ("minimal", "low", "medium" or "high", depending on the model), while other models like Gemini 2.5 use `thinking_budget` (token count). See [Gemini Thinking Documentation](https://ai.google.dev/gemini-api/docs/thinking) for details. - **Multimodal Input Support** Accepts both text and image data for more expressive interactions with configurable image optimization. @@ -220,7 +220,9 @@ GOOGLE_STRIP_THINKING_FROM_HISTORY=true GOOGLE_THINKING_BUDGET=-1 # Thinking level for Gemini 3 models only -# Most Gemini 3 models accept "low" or "high" +# gemini-3-pro accepts "low" or "high"; gemini-3-flash all four ("minimal", "low", +# "medium", "high"); gemini-3.1-pro "low", "medium" or "high"; other Gemini 3 text +# models "low" or "high" (see "Thinking Levels" below for the full list) # gemini-3.1-flash-image(-preview) and gemini-3.1-flash-lite-image accept "minimal" or "high" # gemini-nano-banana-2.1 accepts "minimal", "medium" or "high" # The pipeline automatically maps unsupported values to the closest supported level @@ -893,7 +895,11 @@ The Google Gemini pipeline supports advanced thinking configuration to control h Gemini 3 models support the `thinking_level` parameter, which controls the depth of reasoning: -- **Most Gemini 3 models**: support **`"low"`** and **`"high"`**. +- **Gemini 3 text models**, per [Google's thinking documentation](https://ai.google.dev/gemini-api/docs/thinking) (with or without a `-preview` / `-latest` suffix): + - `gemini-3-pro`: **`"low"`** and **`"high"`**. + - `gemini-3-flash`, `gemini-3.5-flash-lite` and `gemini-3.6-flash`: **`"minimal"`**, **`"low"`**, **`"medium"`** and **`"high"`**. + - `gemini-3.1-pro` and `gemini-3.8-flash`: **`"low"`**, **`"medium"`** and **`"high"`**. + - Other Gemini 3 text models: **`"low"`** and **`"high"`**, which every model above supports. - **`gemini-3.1-flash-image`**, **`gemini-3.1-flash-image-preview`** and **`gemini-3.1-flash-lite-image`**: support **`"minimal"`** and **`"high"`** (model default: `"minimal"`). - **`gemini-nano-banana-2.1`**: supports **`"minimal"`**, **`"medium"`** and **`"high"`** (model default: `"medium"`). - **Other Nano Banana IDs and models in `GOOGLE_IMAGE_GENERATION_MODELS`**: the configured level is sent unchanged, because the pipeline does not know their supported levels. @@ -901,7 +907,7 @@ Gemini 3 models support the `thinking_level` parameter, which controls the depth > [!Note] > Gemini 3 models use `thinking_level` and do **not** use `thinking_budget`. The thinking budget setting is ignored for Gemini 3 models. -If you configure an unsupported value for a specific model, the pipeline automatically falls back to the closest supported thinking level instead of sending an invalid API request. +If you configure an unsupported value for a specific model, the pipeline automatically falls back to the closest supported thinking level instead of sending an invalid API request (for example `"medium"` becomes `"high"` on `gemini-3-pro`, `"minimal"` becomes `"low"` on `gemini-3.1-pro`). Set via environment variable: @@ -918,7 +924,7 @@ GOOGLE_THINKING_LEVEL="minimal" #### Per-Chat Override (Reasoning Effort) -The per-chat `reasoning_effort` value can override the environment-level `GOOGLE_THINKING_LEVEL` setting. When a chat specifies a `reasoning_effort` value (for example, `"low"`, `"minimal"`, or `"high"`), it takes precedence over the global environment setting. This allows users to customize reasoning depth on a per-conversation basis. +The per-chat `reasoning_effort` value can override the environment-level `GOOGLE_THINKING_LEVEL` setting. When a chat specifies a `reasoning_effort` value (for example, `"low"`, `"minimal"`, or `"high"`), it takes precedence over the global environment setting. This allows users to customize reasoning depth on a per-conversation basis. A value that is not one of the four level names (for example a number) is ignored with a warning, and the other thinking settings of the request still apply. **Example API Usage:** @@ -965,7 +971,7 @@ GOOGLE_THINKING_BUDGET=1024 #### Per-Chat Override (thinking_budget) -Similar to `reasoning_effort` for Gemini 3 models, the per-chat `thinking_budget` value can override the environment-level `GOOGLE_THINKING_BUDGET` setting. When a chat request includes a `thinking_budget` value, it takes precedence over the global environment setting. This allows users to customize the thinking budget on a per-conversation basis. +Similar to `reasoning_effort` for Gemini 3 models, the per-chat `thinking_budget` value can override the environment-level `GOOGLE_THINKING_BUDGET` setting. When a chat request includes a `thinking_budget` value, it takes precedence over the global environment setting. This allows users to customize the thinking budget on a per-conversation basis. A number sent as a string (`"1024"`) is accepted; a value that is not a number is ignored with a warning, and `GOOGLE_THINKING_BUDGET` applies. **Example API Usage:** diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 270f6c7..1489fb1 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -467,9 +467,12 @@ class Valves(BaseModel): ) THINKING_LEVEL: str = Field( default=os.getenv("GOOGLE_THINKING_LEVEL", ""), - description="Thinking level for Gemini 3 models. Most Gemini 3 models support 'low'/'high', " - "while gemini-3.1-flash-image and gemini-3.1-flash-lite-image support 'minimal'/'high' " - "and gemini-nano-banana-2.1 supports 'minimal'/'medium'/'high'. " + description="Thinking level for Gemini 3 models ('minimal', 'low', 'medium' or 'high'). " + "The supported levels depend on the model: gemini-3-pro 'low'/'high', gemini-3-flash " + "all four, gemini-3.1-pro 'low'/'medium'/'high', gemini-3.1-flash-image and " + "gemini-3.1-flash-lite-image 'minimal'/'high', gemini-nano-banana-2.1 " + "'minimal'/'medium'/'high', other Gemini 3 models 'low'/'high'. An unsupported " + "level is mapped to the closest supported one. " "Ignored for other models. Empty string means use model default.", ) USE_VERTEX_AI: bool = Field( @@ -1382,6 +1385,19 @@ def get_google_models(self, force_refresh: bool = False) -> List[Dict[str, str]] # e.g. gemini-nano-banana-2.10. _NANO_BANANA_2_1_RE = re.compile(r"gemini-nano-banana-2\.1(?:[-@]|$)") + # thinking_level values per Gemini 3 text model, by the model name without a + # "-preview" / "-latest" suffix (https://ai.google.dev/gemini-api/docs/thinking). + # Other Gemini 3 text models get "low"/"high", which every listed one supports. + _THINKING_LEVELS = { + "gemini-3-pro": ["low", "high"], + "gemini-3-flash": ["minimal", "low", "medium", "high"], + "gemini-3.1-pro": ["low", "medium", "high"], + "gemini-3.5-flash-lite": ["minimal", "low", "medium", "high"], + "gemini-3.6-flash": ["minimal", "low", "medium", "high"], + "gemini-3.8-flash": ["low", "medium", "high"], + } + _MODEL_SUFFIX_RE = re.compile(r"-(?:preview|latest)(?:-[\w.-]*)?$") + # Image models whose model pages list Search grounding as "Not supported" # (https://ai.google.dev/gemini-api/docs/models), incl. their preview IDs. _NO_SEARCH_GROUNDING_MODELS = ( @@ -1579,6 +1595,12 @@ def _get_supported_thinking_levels(self, model_id: str) -> List[str]: return ["minimal", "high"] if self._is_gemini_3_family_model(model_id): + if "image" not in model_lower: + levels = self._THINKING_LEVELS.get( + self._MODEL_SUFFIX_RE.sub("", model_lower) + ) + if levels: + return list(levels) return ["low", "high"] return [] @@ -1631,6 +1653,12 @@ def _validate_thinking_level(self, level: str, model_id: str = "") -> Optional[s """ if not level: return None + if not isinstance(level, str): + self.log.warning( + f"Invalid thinking level {level!r}: expected a string. " + "Falling back to model default." + ) + return None normalized = level.strip().lower() valid_levels = ["minimal", "low", "medium", "high"] @@ -1660,16 +1688,33 @@ def _validate_thinking_level(self, level: str, model_id: str = "") -> Optional[s ) return None - def _validate_thinking_budget(self, budget: int) -> int: + def _validate_thinking_budget(self, budget: Any) -> Optional[int]: """ Validate and normalize the thinking budget value. Args: - budget: The thinking budget integer to validate + budget: The thinking budget to validate: an integer, or a number + given as float or string (e.g. "1024" from an API client) Returns: - Validated budget: -1 for dynamic, 0 to disable, or 1-32768 for fixed limit + Validated budget: -1 for dynamic, 0 to disable, or 1-32768 for fixed + limit; None for a value that is not a number """ + if isinstance(budget, str): + try: + budget = float(budget.strip()) + except ValueError: + budget = None + try: + if isinstance(budget, bool) or not isinstance(budget, (int, float)): + raise ValueError + budget = int(budget) # ValueError / OverflowError for nan / inf + except (ValueError, OverflowError): + self.log.warning( + f"Invalid thinking budget {budget!r}: expected a number. Ignoring it." + ) + return None + # -1 means dynamic thinking (let the model decide) if budget == -1: return -1 diff --git a/tests/e2e/suites/gemini.py b/tests/e2e/suites/gemini.py index 4f1fab2..765463a 100644 --- a/tests/e2e/suites/gemini.py +++ b/tests/e2e/suites/gemini.py @@ -747,6 +747,62 @@ def replayed(entry: dict) -> list: f"pro3={think_pro} image={think_img}", ) + # thinking levels per model (https://ai.google.dev/gemini-api/docs/thinking) + await _set( + t, + THINKING_LEVEL="", + MODEL_ADDITIONAL="gemini-3-flash-preview,gemini-3.1-pro-preview", + ) + levels = {} + for name, effort in ( + ("gemini-3-flash-preview", "medium"), + ("gemini-3-flash-preview", "minimal"), + ("gemini-3.1-pro-preview", "medium"), + ("gemini-3.1-pro-preview", "minimal"), + ): + await mock.reset() + await t.owui.chat(f"{FID}.{name}", "Hi", stream=False, reasoning_effort=effort) + think = _gc(await mock.last(_gen)).get("thinkingConfig") or {} + levels[f"{name}:{effort}"] = str(think.get("thinkingLevel", "")).upper() + t.check( + "thinking.level-models", + "levels per model: gemini-3-flash medium -> MEDIUM, minimal -> MINIMAL; " + "gemini-3.1-pro medium -> MEDIUM, minimal -> LOW (closest supported)", + levels + == { + "gemini-3-flash-preview:medium": "MEDIUM", + "gemini-3-flash-preview:minimal": "MINIMAL", + "gemini-3.1-pro-preview:medium": "MEDIUM", + "gemini-3.1-pro-preview:minimal": "LOW", + }, + f"levels={levels}", + ) + + # invalid per-chat values are ignored one by one, the rest of the thinking + # config stays (API clients can send any JSON type) + await mock.reset() + await t.owui.chat(PRO3, "Hi", stream=False, reasoning_effort=5) + think_num = _gc(await mock.last(_gen)).get("thinkingConfig") or {} + await mock.reset() + await t.owui.chat(TEXT, "Hi", stream=False, thinking_budget="2048") + think_str = _gc(await mock.last(_gen)).get("thinkingConfig") or {} + await mock.reset() + await t.owui.chat(TEXT, "Hi", stream=False, thinking_budget="lots") + think_bad = _gc(await mock.last(_gen)).get("thinkingConfig") or {} + await _set(t, MODEL_ADDITIONAL="") + t.check( + "thinking.params-robust", + "reasoning_effort=5 -> model default level, thoughts still included; " + "thinking_budget='2048' -> 2048; thinking_budget='lots' -> THINKING_BUDGET " + "(-1), thoughts still included", + think_num.get("includeThoughts") is True + and "thinkingLevel" not in think_num + and think_str.get("thinkingBudget") == 2048 + and think_bad.get("thinkingBudget") == -1 + and think_bad.get("includeThoughts") is True, + f"effort5={think_num} budget_str={think_str} budget_bad={think_bad}", + ) + await _set(t, THINKING_LEVEL="", INCLUDE_THOUGHTS=False) await mock.reset() r = await t.owui.chat(TEXT, "Hi", stream=False) From df3bb704a0790e99c8ec383e7d15f64e8ee4175c Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:05:18 +0200 Subject: [PATCH 09/12] fix(gemini): shared chat access fixes of the security review - A file named by a message of another user (markdown link or file entry in the saved chain) is read only when that author may read it too, not only the user whose turn it is. Author of a user message: its user_id (Open WebUI 0.12 stores it for every message), else the chat owner, as Open WebUI reads it; of an answer: the author of its parent user message; unknown author: the file is not read. The requesting user's own messages, the current message and API histories keep the requester's rights. Up to Open WebUI 0.11 an admin continuing a user's chat no longer reads the admin's files named in the user's messages. - Generated images (and Veo videos) are linked to a chat only when the user may continue it (_chat_to_link); the decision of the image history is reused within the request. - At most 4 x IMAGE_HISTORY_MAX_REFERENCES (at least 8) images of earlier messages are looked at per request; accounts are fetched once per request (_RequestAccess). - _may_continue_chat allows only the chat itself, not any truthy value. Tests: gemini.shared.deputy, .history-cap, .link-chat, .pipe-truthy (the test filter gains e2e-link and e2e-chat-truthy); imgedit.admin on 0.11. +semver: none Co-Authored-By: Claude Opus 5.5 --- pipelines/google/google_gemini.py | 367 +++++++++++++++++++++++------ tests/e2e/suites/_gemini_shared.py | 274 +++++++++++++++++++-- tests/e2e/suites/gemini.py | 28 ++- 3 files changed, 574 insertions(+), 95 deletions(-) diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 1489fb1..dbd62d4 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -17,8 +17,8 @@ - Retries of temporary API errors for streaming and non-streaming requests - Advanced multimodal input support (text and images) - Unified image generation and editing with Gemini 2.5 Flash Image Preview - - Image editing across turns: earlier generated and uploaded images of a saved chat are sent with the edit request (the newest kept within the image limit, each image once; only files the requesting user may read) - - Shared chats with "Allow replies" (Open WebUI 0.12+): participants edit the images of the chat too; who may continue a chat and read its files follows Open WebUI's own access checks, generated images are linked to the chat like Open WebUI's own + - Image editing across turns: earlier generated and uploaded images of a saved chat are sent with the edit request (the newest kept within the image limit, each image once; only files the requesting user, and the author of the message that names them, may read) + - Shared chats with "Allow replies" (Open WebUI 0.12+): participants edit the images of the chat too; who may continue a chat and read its files follows Open WebUI's own access checks, a file named in another participant's message is used only if that participant may read it too, generated images are linked to the chat (only one the user may continue) like Open WebUI's own - Nano Banana image models (gemini-3.1-flash-image, gemini-3.1-flash-lite-image, gemini-nano-banana-2.1) - Extra image generation model IDs configurable without a code change - Interim thought images skipped, so each generated image is uploaded once (the last one is kept if no final image arrives) @@ -234,6 +234,41 @@ def _unload_stale_modules() -> None: # prefix) and channel messages. Their history exists only in the request. UNSAVED_CHAT_ID_PREFIXES = ("temporary:", "local:", "channel:") +# Images of earlier messages looked at per request, at most this many per +# image IMAGE_HISTORY_MAX_REFERENCES allows (and at least the minimum): a file +# that is skipped (not readable, a duplicate, missing) does not fill the limit, +# so without this cap every link in a chat would be checked on every turn. +IMAGE_HISTORY_CANDIDATES_PER_IMAGE = 4 +IMAGE_HISTORY_MIN_CANDIDATES = 8 + + +class _RequestAccess: + """What the access checks of one request looked up. A local of that + request, never an attribute of the Pipe (it serves concurrent requests). + + ``accounts``: Users.get_user_by_id per user id (the requesting user and the + authors of the messages that name files), fetched once per request. + ``chats``: per chat id whether the requesting user may continue it and + the chat's owner (see Pipe._may_continue_chat), decided once per request, + so linking the generated images reuses the decision of the image history. + """ + + def __init__(self) -> None: + self.accounts: Dict[str, Any] = {} + self.chats: Dict[str, Tuple[bool, Optional[str]]] = {} + + async def account(self, user_id: str) -> Any: + """The user's account (None if there is none); errors propagate.""" + if user_id not in self.accounts: + self.accounts[user_id] = await Users.get_user_by_id(user_id) + return self.accounts[user_id] + + def chat_owner(self, chat_id: str) -> Optional[str]: + """Owner of a chat the requesting user may continue (else None).""" + allowed, owner = self.chats.get(chat_id, (False, None)) + return owner if allowed else None + + # Open WebUI's Tool Search (0.12+, utils/tool_search.py): the tool that returns # the definitions of the tools left out of body["tools"]. TOOL_SEARCH_NAME = "search_tools" @@ -637,29 +672,61 @@ class Valves(BaseModel): # ---------------- Internal Helpers ---------------- # async def _collect_history_images( self, - sources: List[List[str]], + sources: List[Tuple[Optional[str], List[str]]], room: int, current: List[Dict[str, Any]], optimization_stats: List[Dict[str, Any]], __user__: Optional[dict], + access: Optional[_RequestAccess] = None, ) -> List[Dict[str, Any]]: """Images of earlier messages for the ``room`` the current message's images leave under IMAGE_HISTORY_MAX_REFERENCES, oldest first. - ``sources`` holds the image URLs of each earlier message, oldest message - first. They are read newest first and only until the room is filled, so - a long chat does not read and re-encode all of its images. With + ``sources`` holds, per earlier message (oldest first), who wrote it and + its image URLs. They are read newest first and only until the room is + filled, so a long chat does not read and re-encode all of its images, + and at most IMAGE_HISTORY_CANDIDATES_PER_IMAGE per image of the limit + are looked at (a skipped image does not fill the room). With IMAGE_DEDUP_HISTORY an image counts once, at its newest place, and an image of the current message is not sent again as history. + + A file named by a message of another user is read only when that user + may read it too (see _fetch_file_as_base64), so a participant of a + shared chat cannot have the pipe read a file with the rights of the + user whose turn it is. Without a known author (None) a message's files + are not read; its data: URLs are part of the chat anyway. """ if room <= 0: return [] + access = access or _RequestAccess() + requester = (__user__ or {}).get("id") or "?" + limit = max(1, self.valves.IMAGE_HISTORY_MAX_REFERENCES) + cap = max( + IMAGE_HISTORY_CANDIDATES_PER_IMAGE * limit, IMAGE_HISTORY_MIN_CANDIDATES + ) + candidates = cap dedup = self.valves.IMAGE_DEDUP_HISTORY seen = {self._image_part_hash(p) for p in current} if dedup else set() picked: List[Dict[str, Any]] = [] - for urls in reversed(sources): + for author, urls in reversed(sources): for url in reversed(urls): - part = await self._load_image_part(url, optimization_stats, __user__) + if candidates <= 0: + self.log.info( + f"Image history: looked at the newest {cap} images of " + f"earlier messages for user {requester}, older ones are " + "left out" + ) + return picked[::-1] + candidates -= 1 + if author is None and not url.startswith("data:"): + self.log.warning( + f"Not reading a file for user {requester}: the author of " + "the message that names it is unknown" + ) + continue + part = await self._load_image_part( + url, optimization_stats, __user__, access=access, author=author + ) if part is None: continue if dedup: @@ -711,6 +778,7 @@ async def _load_saved_chat_chain( self, __metadata__: Optional[Dict[str, Any]], __user__: Optional[dict], + access: Optional[_RequestAccess] = None, ) -> Optional[List[Dict[str, Any]]]: """The messages of a saved chat from the first one to the current user message (the last entry), as Open WebUI stores them, files included. @@ -736,7 +804,9 @@ async def _load_saved_chat_chain( ): return None try: - if not await self._may_continue_chat(chat_id, user): + if not await self._may_continue_chat( + chat_id, user, access or _RequestAccess() + ): return None messages_map = await Chats.get_messages_map_by_chat_id(chat_id) chain = get_message_list(messages_map or {}, user_message_id) @@ -746,9 +816,40 @@ async def _load_saved_chat_chain( return [m for m in chain if isinstance(m, dict)] or None @staticmethod - async def _may_continue_chat(chat_id: str, user: Dict[str, Any]) -> bool: + def _message_authors( + chain: List[Dict[str, Any]], owner: Optional[str] + ) -> List[Optional[str]]: + """Who wrote each message of a saved chain (None when unknown), whose + access then counts for the files the message names. + + A user message: its ``user_id``. Open WebUI 0.12 stores it with every + message (insert_chat_turn sets it to the user who sent the message, + also for the participants of a shared chat); where it is missing + (Open WebUI up to 0.11 leaves it out of the message map, messages + saved before) the chat's owner, as Open WebUI itself reads it + (``message.get("user_id") or chat.user_id``). Up to 0.11 only the owner + and admins write in a chat. An answer: the author of the user message + it answers, whose prompt it was made for (its parent, the message + before it in the chain: get_message_list follows parentId); None when + that is not a user message. + """ + authors: List[Optional[str]] = [] + for index, msg in enumerate(chain): + if msg.get("role") == "user": + authors.append(str(msg.get("user_id") or owner or "") or None) + elif index and chain[index - 1].get("role") == "user": + authors.append(authors[index - 1]) + else: + authors.append(None) + return authors + + @staticmethod + async def _may_continue_chat( + chat_id: str, user: Dict[str, Any], access: _RequestAccess + ) -> bool: """Whether ``user`` may continue the saved chat, so that its history - (files of other users included) may be read for the image history. + (files of other users included) may be read for the image history and + generated files may be linked to it. With Open WebUI's own checks (_shared_chat_access, Open WebUI 0.12+) exactly what Open WebUI asks before it calls the pipe: @@ -756,22 +857,88 @@ async def _may_continue_chat(chat_id: str, user: Dict[str, Any]) -> bool: which allows the owner and the participants of a chat or folder shared with "Allow replies" (users or groups), but an admin only like any other user (read access to other users' chats does not allow - continuing them). Otherwise the owner or an admin, as Open WebUI up to - 0.11. Errors propagate; the caller refuses then. + continuing them). Only the chat itself allows: any other value, also a + truthy one, refuses. Otherwise the owner or an admin, as Open WebUI up + to 0.11. + + The decision and the chat's owner (see _message_authors) are kept in + ``access`` for the rest of the request. Errors propagate (and are not + kept); the caller refuses then. """ + if chat_id in access.chats: + return access.chats[chat_id][0] + allowed, owner = False, None if OWUI_SHARED_CHAT_ACCESS: - account = await Users.get_user_by_id(user["id"]) - if account is None: - return False - chat = await Chats.get_accessible_chat_by_id( - chat_id, account, permission="write" + account = await access.account(user["id"]) + if account is not None: + chat = await Chats.get_accessible_chat_by_id( + chat_id, account, permission="write" + ) + allowed = getattr(chat, "id", None) == chat_id + owner = getattr(chat, "user_id", None) if allowed else None + elif await Chats.is_chat_owner(chat_id, user["id"]): + allowed, owner = True, user["id"] + elif user.get("role") == "admin": + # an admin may continue any chat; the owner's id is needed for + # the messages without user_id (see _message_authors) + chat = await Chats.get_chat_by_id(chat_id) + allowed = getattr(chat, "id", None) == chat_id + owner = getattr(chat, "user_id", None) if allowed else None + access.chats[chat_id] = (allowed, str(owner) if owner else None) + return allowed + + async def _chat_to_link( + self, + __metadata__: Optional[Dict[str, Any]], + access: Optional[_RequestAccess] = None, + ) -> Optional[Tuple[str, str, str]]: + """(chat id, message id, user id) to link generated files to, or None. + + Saved chats only, and only a chat the requesting user may continue + (_may_continue_chat, decided once per request): Open WebUI's + insert_chat_files checks that the user may read the files, not that + the user may write in the chat, and a linked file can be read by + everyone with access to the chat. + """ + metadata = __metadata__ or {} + chat_id = str(metadata.get("chat_id") or "") + message_id = str(metadata.get("message_id") or "") + user_id = str(metadata.get("user_id") or "") + if ( + not chat_id + or not message_id + or not user_id + or chat_id.startswith(UNSAVED_CHAT_ID_PREFIXES) + ): + return None + access = access or _RequestAccess() + try: + account = await access.account(user_id) + allowed = account is not None and await self._may_continue_chat( + chat_id, + {"id": user_id, "role": getattr(account, "role", None)}, + access, ) - return bool(chat) - if user.get("role") == "admin": - return True - return bool(await Chats.is_chat_owner(chat_id, user["id"])) + except Exception as e: + self.log.warning( + f"Not linking generated files to chat {chat_id}: the access " + f"check failed: {type(e).__name__}" + ) + return None + if not allowed: + self.log.warning( + f"Not linking generated files to chat {chat_id}: user {user_id} " + "may not continue it" + ) + return None + return chat_id, message_id, user_id - async def _may_read_file(self, file_obj: Any, user: Dict[str, Any]) -> bool: + async def _may_read_file( + self, + file_obj: Any, + user: Dict[str, Any], + access: Optional[_RequestAccess] = None, + ) -> bool: """Whether ``user`` may read an Open WebUI file (``file_obj``, whose id a message may name freely, e.g. in a markdown link of an API client). @@ -779,18 +946,24 @@ async def _may_read_file(self, file_obj: Any, user: Dict[str, Any]) -> bool: admin and, with Open WebUI's own checks (_shared_chat_access, Open WebUI 0.12+), whoever has_access_to_file(id, "read", user) lets read the file (e.g. the participants of a shared chat it is attached to, the members - of a knowledge base or channel). Errors refuse, and the file id stays - out of the log. + of a knowledge base or channel). ``user`` is ``__user__`` or, for the + author of a message, ``{"id": ...}`` (the role then comes from the + account, fetched once per request with ``access``). Errors refuse, and + the file id stays out of the log. """ user_id = user.get("id") if (user_id and file_obj.user_id == user_id) or user.get("role") == "admin": return True - if not user_id or not OWUI_SHARED_CHAT_ACCESS: + if not user_id or ("role" in user and not OWUI_SHARED_CHAT_ACCESS): return False try: - account = await Users.get_user_by_id(user_id) + account = await (access or _RequestAccess()).account(user_id) if account is None: return False + if getattr(account, "role", None) == "admin": + return True + if not OWUI_SHARED_CHAT_ACCESS: + return False allowed = await owui_file_access.has_access_to_file( file_obj.id, "read", account ) @@ -982,15 +1155,21 @@ async def _build_image_generation_contents( __event_emitter__: Optional[Callable], __metadata__: Optional[Dict[str, Any]] = None, __user__: Optional[dict] = None, + access: Optional[_RequestAccess] = None, ) -> Tuple[List[Dict[str, Any]], Optional[str]]: """Construct the contents payload for image-capable models. The prompt and the current images come from the last user message of the request. Images of earlier turns come from the saved chat when there is one (see _saved_message_image_urls), else from the request. + The files a saved message names are read for its author too (see + _message_authors and _collect_history_images); the request's messages + are the requesting user's own. Returns tuple (contents, system_instruction) where system_instruction is extracted from system messages. """ + access = access or _RequestAccess() + requester = (__user__ or {}).get("id") # Extract user-defined system instruction first user_system_instruction = next( (msg["content"] for msg in messages if msg.get("role") == "system"), @@ -1008,11 +1187,15 @@ async def _build_image_generation_contents( optimization_stats: List[Dict[str, Any]] = [] prompt, current_images = await self._extract_images_from_message( - last_user_msg, stats_list=optimization_stats, __user__=__user__ + last_user_msg, + stats_list=optimization_stats, + __user__=__user__, + access=access, ) current_images = self._deduplicate_images(current_images) - chain = await self._load_saved_chat_chain(__metadata__, __user__) + sources: List[Tuple[Optional[str], List[str]]] + chain = await self._load_saved_chat_chain(__metadata__, __user__, access) if chain: # The chain ends with the saved current user message. Open WebUI's # guided regeneration appends the guidance as one more user @@ -1026,13 +1209,16 @@ async def _build_image_generation_contents( self._trailing_user_messages(messages[: end + 1]) == self._trailing_user_messages(chain, saved=True) + 1 ) + authors = self._message_authors( + chain, access.chat_owner(str((__metadata__ or {}).get("chat_id"))) + ) sources = [ - self._saved_message_image_urls(m) - for m in (chain if guided else chain[:-1]) + (authors[i], self._saved_message_image_urls(m)) + for i, m in enumerate(chain if guided else chain[:-1]) ] else: sources = [ - self._content_image_sources(m.get("content", ""))[1] + (requester, self._content_image_sources(m.get("content", ""))[1]) for m in messages if m is not last_user_msg and m.get("role") in {"user", "assistant"} ] @@ -1043,6 +1229,7 @@ async def _build_image_generation_contents( current_images, optimization_stats, __user__, + access, ) combined, reused_flags = self._apply_order_and_limit( @@ -1975,6 +2162,7 @@ async def _link_chat_files( self, files: List[Dict[str, Any]], __metadata__: Optional[Dict[str, Any]], + access: Optional[_RequestAccess] = None, ) -> None: """Link uploaded generated files to their chat message (chat_file rows). @@ -1984,19 +2172,9 @@ async def _link_chat_files( participants of a chat shared with "Allow replies" (and the viewers of a shared chat) may then read the images, in the web UI and in the image history of their turns. Open WebUI links a file only when the user may - read it. Saved chats only. + read it; the pipeline links only to a saved chat the user may continue + (see _chat_to_link; ``access`` holds the decision of the image history). """ - metadata = __metadata__ or {} - chat_id = str(metadata.get("chat_id") or "") - message_id = metadata.get("message_id") - user_id = metadata.get("user_id") - if ( - not chat_id - or not message_id - or not user_id - or chat_id.startswith(UNSAVED_CHAT_ID_PREFIXES) - ): - return file_ids = [] for entry in files: match = re.search(r"/files/([A-Za-z0-9_-]+)", str(entry.get("url") or "")) @@ -2004,6 +2182,10 @@ async def _link_chat_files( file_ids.append(match.group(1)) if not file_ids: return + target = await self._chat_to_link(__metadata__, access) + if target is None: + return + chat_id, message_id, user_id = target try: await Chats.insert_chat_files( chat_id=chat_id, @@ -2117,6 +2299,7 @@ async def _append_generated_images( generated_image_files: List[Dict[str, Any]], __event_emitter__: Optional[Callable], __metadata__: Optional[Dict[str, Any]] = None, + access: Optional[_RequestAccess] = None, ) -> str: """Attach collected images to the message and return the final content. @@ -2130,7 +2313,7 @@ async def _append_generated_images( generated_image_files, __event_emitter__ ) if files_emitted: - await self._link_chat_files(generated_image_files, __metadata__) + await self._link_chat_files(generated_image_files, __metadata__, access) if generated_image_files and not files_emitted: generated_images.extend( @@ -3034,12 +3217,15 @@ async def _extract_images_from_message( *, stats_list: Optional[List[Dict[str, Any]]] = None, __user__: Optional[dict] = None, + access: Optional[_RequestAccess] = None, ) -> Tuple[str, List[Dict[str, Any]]]: """Extract prompt text and ALL images from a single user message. This replaces the previous single-image _find_image logic for image-capable - models so that multi-image prompts are respected. Open WebUI files are - read only when they belong to ``__user__`` (see _fetch_file_as_base64). + models so that multi-image prompts are respected. The message is the + requesting user's own (the request's last user message): Open WebUI + files are read only when ``__user__`` may read them (see + _fetch_file_as_base64). Returns: (prompt_text, image_parts) @@ -3053,8 +3239,9 @@ async def _extract_images_from_message( ) text_segments, urls = self._content_image_sources(content) image_parts: List[Dict[str, Any]] = [] + access = access or _RequestAccess() for url in urls: - part = await self._load_image_part(url, stats_list, __user__) + part = await self._load_image_part(url, stats_list, __user__, access=access) if part: image_parts.append(part) @@ -3095,14 +3282,20 @@ async def _load_image_part( url: str, stats_list: Optional[List[Dict[str, Any]]], __user__: Optional[dict], + *, + access: Optional[_RequestAccess] = None, + author: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """The inline_data part of one image URL (a data: URL, or an Open WebUI - file the user may read), optimized in a worker thread so that decoding - large images does not block the event loop; None if it is unreadable.""" + file the user, and the ``author`` of the message that names it, may + read), optimized in a worker thread so that decoding large images does + not block the event loop; None if it is unreadable.""" if url.startswith("data:"): data_url: Optional[str] = url else: - data_url = await self._fetch_file_as_base64(url, __user__) + data_url = await self._fetch_file_as_base64( + url, __user__, access=access, author=author + ) if not data_url: return None try: @@ -3337,7 +3530,12 @@ def _optimize_image_for_api( return f"data:image/jpeg;base64,{encoded if 'encoded' in locals() else image_data}" async def _fetch_file_as_base64( - self, file_url: str, __user__: Optional[dict] = None + self, + file_url: str, + __user__: Optional[dict] = None, + *, + access: Optional[_RequestAccess] = None, + author: Optional[str] = None, ) -> Optional[str]: """ Fetch a file from Open WebUI's file system and convert to base64. @@ -3346,11 +3544,19 @@ async def _fetch_file_as_base64( the user's own files, every file for an admin, and on Open WebUI 0.12+ the files Open WebUI lets the user read, e.g. those of a chat shared with the user): a message can name any file id, for example in a - markdown link sent by an API client. + markdown link sent by an API client. A file named by a message of + another user (``author``) is read only when that user may read it too: + otherwise a participant of a shared chat could name a file only the + requesting user may read, and the pipe would send it to Gemini on that + user's turn (and link the image made from it to the chat). Args: file_url: File URL from Open WebUI __user__: The requesting user (without one no file is read) + access: What the access checks of this request looked up + author: Who wrote the message that names the file, when that is + not the requesting user's own message (None: the requesting + user's own message, e.g. the current one) Returns: Base64 encoded file data or None if file not found or not readable @@ -3366,16 +3572,27 @@ async def _fetch_file_as_base64( file_obj = await Files.get_file_by_id(fid) user = __user__ or {} - if file_obj and not await self._may_read_file(file_obj, user): + access = access or _RequestAccess() + refused = "" + if file_obj and not await self._may_read_file(file_obj, user, access): + refused = "it does not belong to the requesting user" + ( + " and Open WebUI grants no read access to it" + if OWUI_SHARED_CHAT_ACCESS + else "" + ) + elif ( + file_obj + and author + and author != user.get("id") + and not await self._may_read_file(file_obj, {"id": author}, access) + ): + refused = ( + f"it is named in a message of user {author}, who may not read it" + ) + if refused: # the file id of another user stays out of the log self.log.warning( - f"Not reading a file for user {user.get('id') or '?'}: it " - "does not belong to the requesting user" - + ( - " and Open WebUI grants no read access to it" - if OWUI_SHARED_CHAT_ACCESS - else "" - ) + f"Not reading a file for user {user.get('id') or '?'}: {refused}" ) return None if file_obj and file_obj.path: @@ -3631,6 +3848,7 @@ async def _upload_video_with_status( __user__: dict, __event_emitter__: Optional[Callable], __metadata__: Optional[Dict[str, Any]] = None, + access: Optional[_RequestAccess] = None, ) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: """Upload video with status updates and data-URL fallback. @@ -3650,9 +3868,11 @@ async def _upload_video_with_status( }, ) - user = await Users.get_user_by_id(__user__["id"]) - chat_id = __metadata__.get("chat_id") if __metadata__ else None - message_id = __metadata__.get("message_id") if __metadata__ else None + access = access or _RequestAccess() + user = await access.account(__user__["id"]) + # linked only to a saved chat the user may continue (_chat_to_link) + target = await self._chat_to_link(__metadata__, access) + chat_id, message_id = (target[0], target[1]) if target else (None, None) video_url, file_entry = await self._upload_video( __request__=__request__, user=user, @@ -5456,11 +5676,14 @@ async def _generate_video( __user__: Optional[dict] = None, __metadata__: Optional[Dict[str, Any]] = None, user: Optional[UserModel] = None, + access: Optional[_RequestAccess] = None, ) -> Union[str, Dict[str, Any], AsyncIterator[Union[str, Dict[str, Any]]]]: """Generate video using Google Veo models (long-running operation with polling). - `user` is the requesting user for the forwarded user info headers. + `user` is the requesting user for the forwarded user info headers; + `access` what the access checks of this request looked up. """ + access = access or _RequestAccess() async def emit_status(description: str, done: bool) -> None: if not __event_emitter__: @@ -5487,7 +5710,7 @@ async def emit_status(description: str, done: bool) -> None: return "Error: No user message found for video generation" prompt, images = await self._extract_images_from_message( - last_user_msg, __user__=__user__ + last_user_msg, __user__=__user__, access=access ) if not prompt: return "Error: No prompt provided for video generation" @@ -5526,6 +5749,7 @@ async def emit_status(description: str, done: bool) -> None: __request__=__request__, __user__=__user__, __metadata__=__metadata__, + access=access, ) finally: # The client is per request: close it after polling and the @@ -5545,6 +5769,7 @@ async def _run_video_generation( __request__: Optional[Request], __user__: Optional[dict], __metadata__: Optional[Dict[str, Any]], + access: Optional[_RequestAccess] = None, ) -> Union[str, Dict[str, Any], AsyncIterator[Union[str, Dict[str, Any]]]]: """Start a Veo operation, poll it, then upload and attach the videos.""" try: @@ -5667,6 +5892,7 @@ async def _run_video_generation( __user__, __event_emitter__, __metadata__, + access, ) else: video_data_b64 = base64.b64encode(video_bytes).decode("utf-8") @@ -5873,8 +6099,11 @@ async def _pipe( # The requesting user for the forwarded user info headers. A local, not # an attribute: the Pipe instance is shared by concurrent requests. user: Optional[UserModel] = None + # What the access checks of this request look up (accounts, whether the + # user may continue the chat), also a local + access = _RequestAccess() if __user__ and self.valves.ENABLE_FORWARD_USER_INFO_HEADERS: - user = await Users.get_user_by_id(__user__["id"]) + user = await access.account(__user__["id"]) # Remember statuses that are still running, to close them if the request # is stopped or fails (see finally). @@ -5902,6 +6131,7 @@ async def _pipe( __user__, __metadata__, user=user, + access=access, ) # Check if this model supports image generation @@ -5930,7 +6160,7 @@ async def _pipe( contents, system_instruction, ) = await self._build_image_generation_contents( - messages, __event_emitter__, __metadata__, __user__ + messages, __event_emitter__, __metadata__, __user__, access ) # For image generation, system_instruction is integrated into the prompt # so it will be None here (this is expected and correct) @@ -6218,6 +6448,7 @@ async def get_response(): generated_image_files, __event_emitter__, __metadata__, + access, ) content = full_response if full_response else NO_CONTENT_TEXT diff --git a/tests/e2e/suites/_gemini_shared.py b/tests/e2e/suites/_gemini_shared.py index 56e11e7..23c555f 100644 --- a/tests/e2e/suites/_gemini_shared.py +++ b/tests/e2e/suites/_gemini_shared.py @@ -24,12 +24,21 @@ with the upload "upload1" (generated image "final"), then B, A and G write in it. Chat Y: two turns of A (upload "upload2", generated "gen1"). +Chat X2 of A, shared with "Allow replies" with B, covers the access fixes of +the security review (see _security): a file B names in a message is read on +A's turn only if B may read it too (shared.deputy), the images of earlier +messages looked at per request are capped (shared.history-cap) and generated +images are linked only to a chat the user may continue (shared.link-chat). + The test-only global filter REDIRECT_FILTER hands the pipe another chat in ``__metadata__`` (``e2e-chat=/`` in the prompt of an -API request), as if Open WebUI had called it for that chat without its own -check: so the pipe's own check is tested also for the users Open WebUI turns -away before the pipe (the admin, the read-only viewer, the outsider). Its -fault markers make the pipe's next call of an access helper raise. +API request; ``e2e-link=`` with a new message id, so the pipe emits +and links its images as for a chat message), as if Open WebUI had called it +for that chat without its own check: so the pipe's own check is tested also +for the users Open WebUI turns away before the pipe (the admin, the read-only +viewer, the outsider). Its fault markers make the pipe's next call of an +access helper raise, or (``e2e-chat-truthy=``) the chat check return True +instead of the chat. On Open WebUI before 0.12 (no "Allow replies") the group records ``shared.unsupported`` (PASS) and ``shared.legacy``: the pipe keeps its owner / @@ -67,6 +76,15 @@ # what the edit request sends with the history of chat X's first turn WITH_HISTORY = [EDIT, "[Image 1]", "upload1", "[Image 2]", "final"] FILE_REFUSED = "does not belong to the requesting user" +# the pipe's refusals of a file (any reason) and of one named by another user +ANY_REFUSED = "Not reading a file for user" +AUTHOR_REFUSED = "who may not read it" +LINK_REFUSED = "Not linking generated files to chat" +FILE_URL = "/api/v1/files/{}/content" +# images of earlier messages the pipe looks at per request: 4 x +# IMAGE_HISTORY_MAX_REFERENCES (5); shared.history-cap plants more links +HISTORY_CAP = 4 * 5 +PLANTED = 30 REDIRECT_FILTER_ID = "e2e_chat_redirect" REDIRECT_FILTER = '''""" @@ -74,39 +92,46 @@ author: owndev version: 0.1.0 license: Apache License 2.0 -description: Test-only filter for tests/e2e (gemini shared). In an API request (no message id), "e2e-chat=/" in the last user message puts that chat id and user message id into the pipe's __metadata__. "e2e-chat-fault=" / "e2e-file-fault=" make the next call of Chats.get_accessible_chat_by_id / has_access_to_file for that id from the Gemini function raise RuntimeError, once (within 60 s). The markers are removed from the message. +description: Test-only filter for tests/e2e (gemini shared). In an API request (no message id), "e2e-chat=/" in the last user message puts that chat id and user message id into the pipe's __metadata__, and "e2e-link=" that chat id and a new message id (the pipe then emits and links its images as for a chat message). "e2e-chat-fault=" / "e2e-file-fault=" make the next call of Chats.get_accessible_chat_by_id / has_access_to_file for that id from the Gemini function raise RuntimeError, "e2e-chat-truthy=" the next call of Chats.get_accessible_chat_by_id return True (a truthy value that is not the chat), once (within 60 s). The markers are removed from the message. """ import functools import re import sys import time +import uuid from typing import Optional REDIRECT = re.compile(r"\\s*e2e-chat=([A-Za-z0-9-]+)/([A-Za-z0-9-]+)") +LINK = re.compile(r"\\s*e2e-link=([A-Za-z0-9-]+)") FAULT = re.compile(r"\\s*e2e-(chat|file)-fault=([A-Za-z0-9-]+)") +TRUTHY = re.compile(r"\\s*e2e-chat-truthy=([A-Za-z0-9-]+)") -def _arm(owner, name: str, key: str, message: str) -> None: +def _arm(owner, name: str, key: str, outcome) -> None: """Wrap owner. once (an async wrapper with the same signature, so the pipe's feature detection still sees the helper; it stays in place, inert - unless armed) and arm it for ``key`` (its first argument).""" + unless armed) and arm it for ``key`` (its first argument): its next call + from the Gemini function raises ``outcome`` (an exception) or returns it.""" wrapper = getattr(owner, name) - if not hasattr(wrapper, "e2e_armed"): + if not hasattr(wrapper, "e2e_outcomes"): original = wrapper @functools.wraps(original) async def wrapper(*args, **kwargs): caller = sys._getframe(1).f_globals.get("__name__", "") - armed = wrapper.e2e_armed + armed = wrapper.e2e_outcomes if caller.startswith("function_gemini") and args and args[0] in armed: - if time.time() < armed.pop(args[0]): - raise RuntimeError(message) + deadline, result = armed.pop(args[0]) + if time.time() < deadline: + if isinstance(result, BaseException): + raise result + return result return await original(*args, **kwargs) - wrapper.e2e_armed = {} + wrapper.e2e_outcomes = {} setattr(owner, name, wrapper) - wrapper.e2e_armed[key] = time.time() + 60 + wrapper.e2e_outcomes[key] = (time.time() + 60, outcome) class Filter: @@ -115,20 +140,29 @@ async def inlet(self, body: dict, __metadata__: Optional[dict] = None) -> dict: text = messages[-1].get("content") if messages else None if __metadata__ is None or not isinstance(text, str): return body + from open_webui.models.chats import Chats + for kind, key in FAULT.findall(text): if kind == "chat": - from open_webui.models.chats import Chats - - _arm(Chats, "get_accessible_chat_by_id", key, "e2e chat access fault") + error = RuntimeError("e2e chat access fault") + _arm(Chats, "get_accessible_chat_by_id", key, error) else: from open_webui.utils.access_control import files - _arm(files, "has_access_to_file", key, "e2e file access fault") - text = FAULT.sub("", text) + error = RuntimeError("e2e file access fault") + _arm(files, "has_access_to_file", key, error) + for key in TRUTHY.findall(text): + _arm(Chats, "get_accessible_chat_by_id", key, True) + text = TRUTHY.sub("", FAULT.sub("", text)) match = REDIRECT.search(text) if match and not __metadata__.get("message_id"): __metadata__["chat_id"], __metadata__["user_message_id"] = match.groups() text = REDIRECT.sub("", text) + link = LINK.search(text) + if link and not __metadata__.get("message_id"): + __metadata__["chat_id"] = link.group(1) + __metadata__["message_id"] = str(uuid.uuid4()) + text = LINK.sub("", text) messages[-1]["content"] = text.strip() return body ''' @@ -222,6 +256,147 @@ def _brief(chat) -> str: return f"HTTP {chat.http_status} done={chat.done}" +async def _file_ids(client) -> set: + """Ids of the user's newest files (the first page of GET /api/v1/files/).""" + _, data = await client.api("GET", "/api/v1/files/?content=false") + items = data.get("items") if isinstance(data, dict) else data + return {f.get("id") for f in items or [] if isinstance(f, dict)} + + +async def _security(t: Suite, mock, owner, part, outsider, private, admin_file): + """Chat X2 of the owner A, shared with "Allow replies" with the participant + B, for the checks of the access fixes in 1.20.0: + + - deputy: B names A's private file in a message of X2. A's next turn must + not send it (B may not read it, so A's rights must not be used for it); + A's own link to it, in A's own message, is sent; + - history cap: B plants PLANTED links to files A may not use (the admin's + file and A's private file); A's next turn checks at most HISTORY_CAP + images of earlier messages; + - linking: the test filter hands the pipe X2 with a new message id; B's + generated image is linked to X2 (A may read it), the outsider C's + images are not (C may not continue X2), with and without the decision of + the image history (``e2e-chat=``). + + Returns what the checks need; ``error`` is set when the flow broke.""" + r: dict = {"error": "", "a3": {}, "a4": {}, "a5": {}, "a7": {}} + try: + upload = await _upload_png(owner, UPLOAD_PNG_1, "upload1.png") + url = FILE_URL.format(private["id"]) + async with BrowserSession(owner) as b: + await mock.reset() + a1 = await b.chat( + IMAGE_GA, + "Draw a certificate like this", + params=LEGACY, + user_files=[upload], + ) + r["shares"] = await _share( + owner, a1.chat_id, [_grant("user", part.user["id"])], replies=True + ) + r["before"] = await _file_status(part, private["id"]) + async with BrowserSession(part) as b: + await mock.reset() + b2 = await b.chat( + IMAGE_GA, + f"Nice ![p]({url}) final-image-2", + params=LEGACY, + chat_id=a1.chat_id, + parent_id=a1.message_id, + ) + r["b2"] = _brief(b2) + await t.log.settle(0.5) + mark = t.mark() + async with BrowserSession(owner) as b: + await mock.reset() + a3 = await b.chat( + IMAGE_GA, + "Make it blue final-image-3", + params=LEGACY, + chat_id=a1.chat_id, + parent_id=b2.message_id, + ) + r["a3"], r["a3_done"] = await _edit_request(mock), a3.done + await t.log.settle(0.5) + r["a3_refused"] = len(t.log.lines(mark, AUTHOR_REFUSED)) + await mock.reset() + a4 = await b.chat( + IMAGE_GA, + f"Use this one ![p]({url}) final-image-1", + params=LEGACY, + chat_id=a1.chat_id, + parent_id=a3.message_id, + ) + r["a4"] = await _edit_request(mock) + await mock.reset() + a5 = await b.chat( + IMAGE_GA, + "Make it red", + params=LEGACY, + chat_id=a1.chat_id, + parent_id=a4.message_id, + ) + r["a5"], r["a5_done"] = await _edit_request(mock), a5.done + r["after"] = await _file_status(part, private["id"]) + + planted = " ".join( + f"![x]({FILE_URL.format(fid)})" + for fid in [admin_file["id"], private["id"]] * (PLANTED // 2) + ) + async with BrowserSession(part) as b: + await mock.reset() + b6 = await b.chat( + IMAGE_GA, + f"Look {planted} final-image-2", + params=LEGACY, + chat_id=a1.chat_id, + parent_id=a5.message_id, + ) + r["b6"] = _brief(b6) + await t.log.settle(0.5) + mark = t.mark() + async with BrowserSession(owner) as b: + await mock.reset() + a7 = await b.chat( + IMAGE_GA, + "Make it green", + params=LEGACY, + chat_id=a1.chat_id, + parent_id=b6.message_id, + ) + r["a7"], r["a7_brief"] = await _edit_request(mock), _brief(a7) + await t.log.settle(0.5) + r["a7_done"] = a7.done + r["cap_refused"] = len(t.log.lines(mark, ANY_REFUSED)) + + before_b, before_c = await _file_ids(part), await _file_ids(outsider) + mark = t.mark() + await mock.reset() + link = f"e2e-link={a1.chat_id}" + r["link_status"] = [ + getattr(await client.chat(IMAGE_GA, text, stream=False), "status", None) + for client, text in ( + (part, f"Draw a star {link}"), + (outsider, f"Draw a star {link}"), + (outsider, f"Draw a star {_target(a5)} {link}"), + ) + ] + await t.log.settle(0.5) + r["link_refused"] = len(t.log.lines(mark, LINK_REFUSED)) + t.expect_warnings(mark, ("function_gemini", LINK_REFUSED)) + r["owner_reads_b"] = [ + await _file_status(owner, fid) + for fid in sorted((await _file_ids(part)) - before_b) + ] + r["owner_reads_c"] = [ + await _file_status(owner, fid) + for fid in sorted((await _file_ids(outsider)) - before_c) + ] + except Exception as exc: # noqa: BLE001 - reported in the checks + r["error"] = f"setup failed: {exc!r} " + return r + + async def legacy(t: Suite, mock, version: str) -> None: """Open WebUI before 0.12: no "Allow replies", and has_access_to_file (0.11) knows only share links with grants. The pipe keeps its owner / admin @@ -322,6 +497,7 @@ async def shared(t: Suite, mock) -> None: denied = chat_fault_logged = file_fault_logged = 0 ids_logged: list = [] group_id = "" + sec: dict = {"error": "not run ", "a3": {}, "a4": {}, "a5": {}, "a7": {}} try: for key, name, email in ( ("owner", "E2E Shared Owner", OWNER), @@ -452,6 +628,8 @@ async def shared(t: Suite, mock) -> None: ("admin", t.owui, _target(cb)), ("viewer", users["viewer"], _target(cy2)), ("outsider", users["outsider"], _target(cb)), + # the chat check answers True (not the chat) for the admin + ("truthy", t.owui, f"{_target(cb)} e2e-chat-truthy={cb.chat_id}"), ): redirected[key] = await _api_edit(client, mock, f"{EDIT} {target}") mark = t.mark() @@ -505,6 +683,9 @@ async def shared(t: Suite, mock) -> None: timeout=180, ) video_image = await _video_image(mock) + sec = await _security( + t, mock, owner, part, users["outsider"], private, admin_file + ) except Exception as exc: # noqa: BLE001 - reported in the checks error = f"setup failed: {exc!r} " finally: @@ -593,6 +774,12 @@ async def shared(t: Suite, mock) -> None: "a user the chat is not shared with: nothing of the chat is read", [EDIT], ), + ( + "truthy", + "the admin, and the chat check answers a truthy value that is not " + "the chat: nothing of the chat is read", + [EDIT], + ), ): result, req = redirected.get(key, (None, empty)) t.check( @@ -647,3 +834,54 @@ async def shared(t: Suite, mock) -> None: ready and getattr(video, "status", None) == 200 and video_image == "upload1", f"{setup} HTTP {getattr(video, 'status', None)} image={video_image!r}", ) + sec_ready = ready and not sec["error"] and sec.get("shares") == [200, 200] + sec_setup = f"{setup} {sec['error']}x2_shares={sec.get('shares')}" + a3, a4, a5, a7 = sec["a3"], sec["a4"], sec["a5"], sec["a7"] + t.check( + "shared.deputy", + "browser, a participant names a private file of the owner in a message " + "of the shared chat: the owner's next turn does not send it (logged), the " + "file stays unreadable for the participant; the owner's own link to it " + "is sent, in the owner's message and later from the history", + sec_ready + and sec.get("before") == 404 + and sec.get("after") == 404 + and sec.get("a3_done") + and a3.get("images") == ["upload1", "final", "gen2"] + and sec.get("a3_refused", 0) > 0 + and "saved" in (a4.get("images") or []) + and sec.get("a5_done") + and a5.get("images") == ["final", "gen2", "gen3", "saved", "gen1"], + f"{sec_setup} participant: private HTTP {sec.get('before')} -> " + f"{sec.get('after')} turn: {sec.get('b2')} owner: images={a3.get('images')} " + f"refused_logged={sec.get('a3_refused')} own link: {a4.get('images')} " + f"then {a5.get('images')}", + ) + t.check( + "shared.history-cap", + f"browser, a participant plants {PLANTED} links to files the owner may " + f"not use: the owner's next turn looks at no more than {HISTORY_CAP} " + "images of earlier messages (4 x IMAGE_HISTORY_MAX_REFERENCES), logs no " + "more refusals and completes", + sec_ready + and sec.get("a7_done") + and 0 < sec.get("cap_refused", 0) <= HISTORY_CAP + and a7.get("images") == ["gen2"], + f"{sec_setup} plant: {sec.get('b6')} owner: {sec.get('a7_brief')} " + f"images={a7.get('images')} refused_logged={sec.get('cap_refused')}", + ) + t.check( + "shared.link-chat", + "API, the pipe called for a chat message of another chat (test filter): " + "a participant's generated image is linked to that chat (the owner may " + "read it), an outsider's are not (logged; with and without the decision " + "of the image history)", + sec_ready + and sec.get("link_status") == [200, 200, 200] + and sec.get("owner_reads_b") == [200] + and sec.get("owner_reads_c") == [404, 404] + and sec.get("link_refused") == 2, + f"{sec_setup} HTTP {sec.get('link_status')} owner reads participant's " + f"{sec.get('owner_reads_b')} outsider's {sec.get('owner_reads_c')} " + f"refused_logged={sec.get('link_refused')}", + ) diff --git a/tests/e2e/suites/gemini.py b/tests/e2e/suites/gemini.py index 765463a..faf97af 100644 --- a/tests/e2e/suites/gemini.py +++ b/tests/e2e/suites/gemini.py @@ -40,10 +40,14 @@ IMAGE_HISTORY_MAX_REFERENCES keeps the current and the newest images, older saved forms (data: URL files, markdown links), temporary chats and database errors use the request, files of - another user are read only for an admin (also on the Veo path) + another user are read only for an admin (also on the Veo path), + a file named in another user's message only if that user may + read it too shared chats shared with "Allow replies" (Open WebUI 0.12): participants - edit with the chat's images, access as Open WebUI decides it - (suites/_gemini_shared.py) + edit with the chat's images, access as Open WebUI decides it, + files named by another participant only with that participant's + access too, a cap on the files checked, links only to chats the + user may continue (suites/_gemini_shared.py) toolsapi native tool calling, API path: client tools -> tool_calls (stream, non-stream, finish_reason), text answers, streaming valve off, malformed / unexpected calls, continuation with signatures and @@ -192,6 +196,7 @@ # shared chats (1.20.0) "Failed to link generated image files", "File access check failed", + "Not linking generated files", ) ) # Valves and UserValves of google_gemini.py 1.16.1 with their defaults (valve @@ -2857,8 +2862,11 @@ async def imgedit_db_error(t: Suite, mock) -> None: async def imgedit_foreign(t: Suite, mock) -> None: """Files of another user are never read: a message can name any file id. - An admin may read them (also when continuing the user's chat, which Open - WebUI allows up to 0.11).""" + An admin may read them in the admin's own requests. When an admin + continues the user's chat (Open WebUI allows it up to 0.11), a file named + in the user's message is read only if the user may read it too (1.20.0): + otherwise the user could have the pipe read the admin's files on the + admin's turn.""" setup_error, grants, denied, video_denied, id_logged = "", [], 0, 0, [] c1 = c2 = c3 = ru = ra = vo = vf = None empty = {"contents": 0, "sent": [], "images": []} @@ -2980,7 +2988,9 @@ async def imgedit_foreign(t: Suite, mock) -> None: # user, permission="write"), which needs the owner, a chat shared with # share_mode "continue" or a shared folder (models/chats.py), and answers # 404, so the pipe is never called. Up to 0.11 it allowed the owner or any - # admin (Chats.is_chat_owner(...) or user.role == "admin"). + # admin (Chats.is_chat_owner(...) or user.role == "admin"). The admin's + # file in the user's turn 1 is not sent then: the user wrote that message + # and may not read the file (1.20.0, the author's access counts too). admin_refused = owui_version >= (0, 12) t.check( "imgedit.admin", @@ -2990,15 +3000,15 @@ async def imgedit_foreign(t: Suite, mock) -> None: "the pipe is not called" if admin_refused else "browser, an admin continues the user's chat: the chat is read, " - "the admin's file of turn 1 and the user's generated image are sent" + "the user's generated image is sent, the admin's file named in the " + "user's turn 1 is not (the user may not read it)" ), not setup_error and ( getattr(c3, "http_status", None) == 404 and r3["contents"] == 0 if admin_refused else getattr(c3, "done", False) - and r3["sent"] - == ["Make it blue", "[Image 1]", "foreign", "[Image 2]", "final"] + and r3["sent"] == ["Make it blue", "[Image 1]", "final"] ), f"{setup_error}owui={'.'.join(map(str, owui_version)) or '?'} " f"turn3: HTTP {getattr(c3, 'http_status', None)} " From 0e90fbf47a8371f068c0271740ab64f4f42ec5cc Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:27:01 +0200 Subject: [PATCH 10/12] docs: whose access counts for files of other participants' messages; shared group checks - google-gemini-integration.md: a file named in another user's message is read only if that author may read it too (author = user_id, else the chat owner; answers = author of their user message), the cap on images of earlier messages, linking only to chats the user may continue, the 0.11 admin case. - testing.md: gemini.shared gains pipe-truthy, deputy, history-cap and link-chat (17 checks on 0.12), the filter markers e2e-link and e2e-chat-truthy, imgedit.admin on 0.11, the results of 3443671, the second mutation round and the counts (580 on v0.12.0-slim, 565 on v0.11.4-slim). - README: the shared chat bullet. +semver: none Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/google-gemini-integration.md | 6 ++-- docs/testing.md | 51 +++++++++++++++++++++++-------- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 47cd337..b07a578 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ The functions include a built-in encryption mechanism for sensitive information: - Provides configurable error handling and timeouts: temporary API errors are retried (`GOOGLE_RETRY_COUNT`) for streaming and non-streaming requests, and every started status (image, video) is closed when a request fails or is stopped. - **Advanced image processing**: Optimized image handling with configurable compression, resizing, and quality settings. - **Configurable parameters**: Environment variables for image optimization (quality, max dimensions, format conversion). -- **Multi-image history**: Image edits across turns: the generated and uploaded images of a saved chat are sent with the edit request (temporary chats send only your own uploads), also for everyone who replies in a chat shared with "Allow replies" (Open WebUI 0.12; who may read a chat or file is decided by Open WebUI's own access checks). Configurable image limit that keeps the current and the newest images, hash-based deduplication, and automatic `[Image N]` labels so the model can reference earlier images. +- **Multi-image history**: Image edits across turns: the generated and uploaded images of a saved chat are sent with the edit request (temporary chats send only your own uploads), also for everyone who replies in a chat shared with "Allow replies" (Open WebUI 0.12; who may read a chat or file is decided by Open WebUI's own access checks, and a file another participant names is used only if that participant may read it too). Configurable image limit that keeps the current and the newest images, hash-based deduplication, and automatic `[Image N]` labels so the model can reference earlier images. - **Image generation (Gemini 3)**: Configurable aspect ratio (for example `16:9` or `1:1`) and resolution (`1K`, `2K`, or `4K`) for Gemini 3 image models, with per-user valve overrides. - **Nano Banana image models**: `gemini-nano-banana-2.1`, `gemini-3.1-flash-image` and `gemini-3.1-flash-lite-image` are detected as image models; each generated image is uploaded once (interim thought images are skipped unless no final image arrives). API clients get generated images and videos as links in the answer. Google Search grounding is left out for the image models without Search support (`gemini-2.5-flash-image`, `gemini-3.1-flash-lite-image`). Newer image models outside Gemini 3 / Nano Banana naming can be added via `GOOGLE_IMAGE_GENERATION_MODELS` without a code change. - **Video generation (Veo)**: Generate videos with Google Veo models (3.1, 3, 2). Configurable aspect ratio, resolution, duration, negative prompt, and person generation controls. Supports text-to-video and image-to-video for all supported Veo models. Videos are automatically uploaded and embedded with playback controls. diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index 20e15f7..8fdbee8 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -432,16 +432,18 @@ Where the images of earlier turns come from: - **Saved chats** (the normal web UI chat): from the chat as Open WebUI stored it. That covers the generated images, which are attached to the answers as files (Open WebUI does not pass the files of earlier answers to a pipe), images you uploaded, images linked in earlier answers (pipeline versions before 1.15.2) and, when you regenerate an answer with guidance, the images of the user message whose answer you regenerate. Background tasks such as titles and follow-up suggestions get no images. - **API clients** (no saved chat): from the request's messages, as `image_url` parts (data URLs or Open WebUI file URLs) or markdown image links in the content, such as the `![Generated Image](/api/v1/files//content)` link the pipeline returns. - **Temporary chats**: Open WebUI does not save them, and the web UI sends only the images of your own messages. A generated image of an earlier turn is therefore not sent. To edit it, download it and attach it to your edit message. -- **Shared chats with "Allow replies"** (Open WebUI 0.12 and later): the people a chat or its folder is shared with in this mode write in the owner's chat, and their edit requests get the images of the whole chat like the owner's: the owner's uploads, the generated images of every answer and the images of the other participants. Since 1.20.0 the pipeline links the images it generates to their chat (on every Open WebUI version), as Open WebUI does for its own image generation and for uploads, so Open WebUI also shows them to the participants and to the people a chat is shared with. Images generated before pipeline 1.20.0 are not linked: Open WebUI shows them only to their owner and admins, and the pipeline does not send them for the other participants. A "Clone only" share or a read-only folder share lets people read the chat but not write in it: Open WebUI refuses their messages before the pipeline runs. +- **Shared chats with "Allow replies"** (Open WebUI 0.12 and later): the people a chat or its folder is shared with in this mode write in the owner's chat, and their edit requests get the images of the whole chat like the owner's: the owner's uploads, the generated images of every answer and the images of the other participants. Since 1.20.0 the pipeline links the images it generates to their chat (on every Open WebUI version), as Open WebUI does for its own image generation and for uploads, so Open WebUI also shows them to the participants and to the people a chat is shared with. It links them only to a chat the user may continue. Images generated before pipeline 1.20.0 are not linked: Open WebUI shows them only to their owner and admins, and the pipeline does not send them for the other participants. A file that another participant names in a message (an image link in the text, an attached file) is sent on your turn only if that participant may read it too, so nobody can have your turn send your private files (see the access rules below). A "Clone only" share or a read-only folder share lets people read the chat but not write in it: Open WebUI refuses their messages before the pipeline runs. Which images are sent: -- `GOOGLE_IMAGE_HISTORY_MAX_REFERENCES` (default 5) limits the images per request. The images of the current message are always kept, the rest of the limit goes to the newest images of earlier turns, and older ones are dropped first. So the image you want to edit stays in the request in a long chat. Earlier images are read newest first and only as many as fit. +- `GOOGLE_IMAGE_HISTORY_MAX_REFERENCES` (default 5) limits the images per request. The images of the current message are always kept, the rest of the limit goes to the newest images of earlier turns, and older ones are dropped first. So the image you want to edit stays in the request in a long chat. Earlier images are read newest first and only as many as fit. An image that is skipped (one the user may not read, a duplicate, a deleted file) does not count toward the limit, so the pipeline looks at no more than 4 images of earlier turns per image of the limit (at least 8; 20 with the default) and leaves older ones out. - Identical images are sent once (`GOOGLE_IMAGE_DEDUP_HISTORY`): an image you attach again (for example a downloaded generated image) is not repeated from the history, and an image of several turns counts at its newest place. Images of earlier turns come before the current message's images unless `GOOGLE_IMAGE_HISTORY_FIRST=false`. - Access follows Open WebUI: the pipeline never reads a chat or file that Open WebUI would not let the requesting user use, also when a message or an API request names the file id of another user. - Open WebUI 0.12 and later (detected by its access checks, not by the version): the history of a saved chat is read when Open WebUI lets the user continue the chat (`Chats.get_accessible_chat_by_id(..., permission="write")`, which Open WebUI itself checks before it calls the pipeline): the owner and the participants of a chat or folder shared with "Allow replies", users or groups. An admin is no exception: Open WebUI 0.12 gives admins read access to other users' chats, not write access. A file is read when it is the user's own, the user is an admin, or Open WebUI's `has_access_to_file` lets the user read it (for example through a shared chat or folder it is attached to, a knowledge base, a channel or a shared model), the same rule as Open WebUI's own file download. - Older Open WebUI versions: only the requesting user's own files and chats (an admin may read every file and chat). + - Whose access counts for a file in the chat's history: the requesting user's, and also the access of the user who wrote the message that names the file, when that is someone else. A participant who names your private file in a message of a shared chat therefore cannot have your next turn send it to Gemini (and get the image made from it, which is linked to the chat). The author of a user message is the user Open WebUI stores with it (`user_id`, set for every message since Open WebUI 0.12), else the chat's owner, as Open WebUI reads it; the author of an answer is the author of the user message it answers. A file whose author is not known is not read. Up to Open WebUI 0.11 only the owner and admins write in a chat: when an admin continues a user's chat there, a file named in the user's messages is read only if the user may read it too. Your own messages, the current message and the messages an API client sends count as yours. - A file the user may not read is skipped and logged without its id. An error in an access check counts as a refusal: such a file is skipped, and for the chat check the images of earlier turns come from the request, as for API clients. + - Generated images and videos are linked to the chat (Open WebUI's `chat_file` links, through which the people with access to the chat may read them) only when the requesting user may continue that chat, by the same check as for the history; Open WebUI's own linking checks only that the user may read the file. - Open WebUI 0.9.6 and later check the images they put into the request themselves in the same way (and also allow files shared with the user). Open WebUI 0.9.0 to 0.9.5 put the image of any file id named in a user message into the request before the pipeline runs, so update Open WebUI if users must not reach each other's files. - Earlier images are sent as parts of your message, which is how Gemini takes input images for editing. Earlier answers are not replayed as model turns, so no thought signatures are needed. diff --git a/docs/testing.md b/docs/testing.md index 2f8707c..43db1d5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -73,7 +73,7 @@ Most of it is waiting: the azure `rag` group adds about 5.5 minutes (query-gener timeouts, the query-generation pause and the 45 s retrieval limit are waited for; in the meta-test, where every mock answers HTTP 500, its requests fail at once), the gemini `tools`, `toolsapi`, `imgedit` and `owuitools` groups about three minutes, the gemini -`owui012` group about one minute, `shared` about 35 s on 0.12, and the filters `ingest` +`owui012` group about one minute, `shared` about one minute on 0.12, and the filters `ingest` group (Logs Ingestion API) another 2.5 minutes. The browser scenarios of the gemini `thinking` group (paced streams for the live block and its duration, Stop while thinking, a tool round) take about 20 s. @@ -146,7 +146,7 @@ events), a **background title task** and that the server log has no unexpected | Suite | Function(s) | Mock | Groups (`--only .`) and file-specific scenarios | | --- | --- | --- | --- | -| `gemini` | `pipelines/google/google_gemini.py` (+ `google_search_tool`) | `mock_gemini.py`, `mock_tools.py` (tool servers, Jupyter) | `models` (image / video indicators, display names #172), `api` (thinking in the `
` block, full usage), `thinking` (summaries not replayed #176: the plain block of chats saved before 1.19.0 and the `type="reasoning"` blocks, also a stopped `done="false"` one; budget, level, include and strip valves; browser path: the live `done="false"` block through throttled `replace` events, no thinking status, the switch to `done="true"` with all thoughts on the first answer part and deltas after it, the duration up to the first answer part (mock trigger `paced-thinking`), Stop while thinking and the next turn without the block (`slow-thinking`), a tool round that clears its live block), `browser`, `tasks` (no `
` in task answers, no grounding tools for the tasks of a web_search chat), `image` (image models forced non-stream, exactly one saved file, the thinking block), `images` (thought images skipped, used as fallback, not used after IMAGE_SAFETY; dedup; two final images; image link for API clients; image history, where the limit keeps the current and the newest images and cuts more current images than it allows; optimization), `nano` (`gemini-nano-banana-2.1`), `imgvalve` (`IMAGE_GENERATION_MODELS`), `imgconfig` (ImageConfig valves, user valve, body), `imgtools` (tools per image model with web_search), `nostream` (`GOOGLE_STREAMING_ENABLED=false` with `stream=true`, #170), `video` (Veo: text and video saved, request shape, image-to-video), `grounding` (`google_search_tool` → googleSearch + urlContext, sources, `[1]` citations, Open WebUI's search statuses (the queries, then `Searched {{count}} sites` with `items` and no `urls`); no grounding without web_search), `vertex` (Vertex AI Search sources), `errors` (400, 500 with retry, blocked prompt, SAFETY finish, image error status, a streamed answer starting with `data:`), `retry` (`RETRY_COUNT` for streams), `status` (Stop leaves no running status), `valves` (model cache vs. valve changes, safety, whitelist, additional models, system prompt, user headers, API version, params, valve names and defaults), `concurrency` (forwarded user headers belong to the requesting user), `streamimg` (inline image in a stream), `imgedit` (image editing across turns, #194: in a saved chat the image of turn 1, attached as a file only, is sent with the edit request on `gemini-3.1-flash-image` and `gemini-nano-banana-2.1`; the follow-up task gets no images; uploads between generated turns in the default mode, with another image per turn (the mock's `final-image-` trigger) so their order shows; guided regeneration, also of an image-only message and of a text contained in the guidance; no guided regeneration after an empty answer, which Open WebUI 0.12 leaves out of the request (`imgedit.empty-answer`, 1.19.1); `IMAGE_HISTORY_MAX_REFERENCES` keeps the upload of the edit message, else the newest images; an image attached again is sent once and counts at its newest place (with `IMAGE_DEDUP_HISTORY=false` it is sent each time); older saved forms (a data: URL image file, markdown links to a file and to a data: URL; a text file is not sent); a temporary chat and a database error (injected by a test-only filter) take the history from the request; an image file of another user in a non-admin user's chat, in a markdown link on the API path and on the Veo path is not read (logged without its id), an admin continuing the user's chat reads it (up to Open WebUI 0.11; 0.12 gives an admin only read access to another user's chat and answers the admin's turn with HTTP 404 before the pipe runs, which `imgedit.admin` checks there instead)), `shared` (chats shared with "Allow replies", Open WebUI 0.12, `google_gemini.py` 1.20.0; see [Shared chats](#shared-chats-geminishared)), `toolsapi` (native tool calling for API clients: client tools → `tool_calls` + `reasoning_details` streamed and non-streamed with `finish_reason` `tool_calls` (also as the openai SDK reads the stream), a text answer to a request with tools (`finish_reason` `stop`), `GOOGLE_STREAMING_ENABLED=false`, `MALFORMED_FUNCTION_CALL` / `UNEXPECTED_TOOL_CALL` streamed and non-streamed, continuation with and without signatures and with `reasoning_details` under `provider_specific_fields`, older turns with a thinking summary, empty arguments, content parts, a numeric id and a `function` that is not an object, `tool_choice` (also an undeclared name and with Search grounding on Gemini 3), name mapping (also a trailing newline) and duplicates, the `default_api.` prefix, schema clean-up, synthetic `owui_` ids, unchanged answers without tools), `tools` (native tool calling through Open WebUI's tool loop: built-in tool streamed and non-streamed, parallel calls (responses in call order), two rounds with summed usage, merged rounds without signatures, text before a call, thinking on / off, workspace Python tool, a tool result with an image, OpenAPI and MCP tool servers, direct tools of the browser, tool approval (approve, reject, two calls), follow-up turns on Gemini 3 / 2.5 / 3 and after an approved call, unknown tool, `MALFORMED_FUNCTION_CALL`, Search grounding with functions on Gemini 3 (the sources of both rounds, the next turn without web search) and grounding only on 2.5, title task, Legacy mode, `builtin_tools` off, `GOOGLE_STREAMING_ENABLED=false`; see [Native tool calling](#native-tool-calling-geminitools-geminitoolsapi)), `owuitools` (Open WebUI's built-in `generate_image`, `edit_image` and `execute_code` called by a Gemini text model: declared only with the chat toggles and for browser sessions, `edit_image` only with image editing on; the image engine `gemini` with `generateContent` and with `predict` (Imagen), editing an upload and the image of the previous turn, the image saved with the answer and its URL sent back to Gemini; a pipe image model makes its own image; the code interpreter with pyodide (the browser runs the code) and with Jupyter; see [Open WebUI's image and code tools](#open-webuis-image-and-code-tools-geminiowuitools)), `owui012` (Open WebUI 0.12 features with native tool calling, version-gated: on 0.11 a missing feature passes with "feature not in Open WebUI "; context compaction between tool rounds (the summary request without tools; afterwards the summary, the user message and the kept rounds with their signatures; a follow-up turn whose boundary falls into the earlier turn), a failed or empty summary request between tool rounds and before a reply (also the summary saved with the chat, also on 0.11) leaves Open WebUI's own summary (1.20.0), Tool Search (`search_tools` and the `` list, the found tool declared and run in the next round and in the next turn, a mapped OpenAPI name; 1.20.0), model controls refused for pipe models and the custom params a control would set, Stream Chat Response off per chat and per model in a tool turn; see [Open WebUI 0.12 features](#open-webui-012-features-geminiowui012)) | +| `gemini` | `pipelines/google/google_gemini.py` (+ `google_search_tool`) | `mock_gemini.py`, `mock_tools.py` (tool servers, Jupyter) | `models` (image / video indicators, display names #172), `api` (thinking in the `
` block, full usage), `thinking` (summaries not replayed #176: the plain block of chats saved before 1.19.0 and the `type="reasoning"` blocks, also a stopped `done="false"` one; budget, level, include and strip valves; browser path: the live `done="false"` block through throttled `replace` events, no thinking status, the switch to `done="true"` with all thoughts on the first answer part and deltas after it, the duration up to the first answer part (mock trigger `paced-thinking`), Stop while thinking and the next turn without the block (`slow-thinking`), a tool round that clears its live block), `browser`, `tasks` (no `
` in task answers, no grounding tools for the tasks of a web_search chat), `image` (image models forced non-stream, exactly one saved file, the thinking block), `images` (thought images skipped, used as fallback, not used after IMAGE_SAFETY; dedup; two final images; image link for API clients; image history, where the limit keeps the current and the newest images and cuts more current images than it allows; optimization), `nano` (`gemini-nano-banana-2.1`), `imgvalve` (`IMAGE_GENERATION_MODELS`), `imgconfig` (ImageConfig valves, user valve, body), `imgtools` (tools per image model with web_search), `nostream` (`GOOGLE_STREAMING_ENABLED=false` with `stream=true`, #170), `video` (Veo: text and video saved, request shape, image-to-video), `grounding` (`google_search_tool` → googleSearch + urlContext, sources, `[1]` citations, Open WebUI's search statuses (the queries, then `Searched {{count}} sites` with `items` and no `urls`); no grounding without web_search), `vertex` (Vertex AI Search sources), `errors` (400, 500 with retry, blocked prompt, SAFETY finish, image error status, a streamed answer starting with `data:`), `retry` (`RETRY_COUNT` for streams), `status` (Stop leaves no running status), `valves` (model cache vs. valve changes, safety, whitelist, additional models, system prompt, user headers, API version, params, valve names and defaults), `concurrency` (forwarded user headers belong to the requesting user), `streamimg` (inline image in a stream), `imgedit` (image editing across turns, #194: in a saved chat the image of turn 1, attached as a file only, is sent with the edit request on `gemini-3.1-flash-image` and `gemini-nano-banana-2.1`; the follow-up task gets no images; uploads between generated turns in the default mode, with another image per turn (the mock's `final-image-` trigger) so their order shows; guided regeneration, also of an image-only message and of a text contained in the guidance; no guided regeneration after an empty answer, which Open WebUI 0.12 leaves out of the request (`imgedit.empty-answer`, 1.19.1); `IMAGE_HISTORY_MAX_REFERENCES` keeps the upload of the edit message, else the newest images; an image attached again is sent once and counts at its newest place (with `IMAGE_DEDUP_HISTORY=false` it is sent each time); older saved forms (a data: URL image file, markdown links to a file and to a data: URL; a text file is not sent); a temporary chat and a database error (injected by a test-only filter) take the history from the request; an image file of another user in a non-admin user's chat, in a markdown link on the API path and on the Veo path is not read (logged without its id), an admin continuing the user's chat reads the chat, but not the admin's file named in the user's message, which the user may not read (up to Open WebUI 0.11; 0.12 gives an admin only read access to another user's chat and answers the admin's turn with HTTP 404 before the pipe runs, which `imgedit.admin` checks there instead)), `shared` (chats shared with "Allow replies", Open WebUI 0.12, `google_gemini.py` 1.20.0; see [Shared chats](#shared-chats-geminishared)), `toolsapi` (native tool calling for API clients: client tools → `tool_calls` + `reasoning_details` streamed and non-streamed with `finish_reason` `tool_calls` (also as the openai SDK reads the stream), a text answer to a request with tools (`finish_reason` `stop`), `GOOGLE_STREAMING_ENABLED=false`, `MALFORMED_FUNCTION_CALL` / `UNEXPECTED_TOOL_CALL` streamed and non-streamed, continuation with and without signatures and with `reasoning_details` under `provider_specific_fields`, older turns with a thinking summary, empty arguments, content parts, a numeric id and a `function` that is not an object, `tool_choice` (also an undeclared name and with Search grounding on Gemini 3), name mapping (also a trailing newline) and duplicates, the `default_api.` prefix, schema clean-up, synthetic `owui_` ids, unchanged answers without tools), `tools` (native tool calling through Open WebUI's tool loop: built-in tool streamed and non-streamed, parallel calls (responses in call order), two rounds with summed usage, merged rounds without signatures, text before a call, thinking on / off, workspace Python tool, a tool result with an image, OpenAPI and MCP tool servers, direct tools of the browser, tool approval (approve, reject, two calls), follow-up turns on Gemini 3 / 2.5 / 3 and after an approved call, unknown tool, `MALFORMED_FUNCTION_CALL`, Search grounding with functions on Gemini 3 (the sources of both rounds, the next turn without web search) and grounding only on 2.5, title task, Legacy mode, `builtin_tools` off, `GOOGLE_STREAMING_ENABLED=false`; see [Native tool calling](#native-tool-calling-geminitools-geminitoolsapi)), `owuitools` (Open WebUI's built-in `generate_image`, `edit_image` and `execute_code` called by a Gemini text model: declared only with the chat toggles and for browser sessions, `edit_image` only with image editing on; the image engine `gemini` with `generateContent` and with `predict` (Imagen), editing an upload and the image of the previous turn, the image saved with the answer and its URL sent back to Gemini; a pipe image model makes its own image; the code interpreter with pyodide (the browser runs the code) and with Jupyter; see [Open WebUI's image and code tools](#open-webuis-image-and-code-tools-geminiowuitools)), `owui012` (Open WebUI 0.12 features with native tool calling, version-gated: on 0.11 a missing feature passes with "feature not in Open WebUI "; context compaction between tool rounds (the summary request without tools; afterwards the summary, the user message and the kept rounds with their signatures; a follow-up turn whose boundary falls into the earlier turn), a failed or empty summary request between tool rounds and before a reply (also the summary saved with the chat, also on 0.11) leaves Open WebUI's own summary (1.20.0), Tool Search (`search_tools` and the `` list, the found tool declared and run in the next round and in the next turn, a mapped OpenAPI name; 1.20.0), model controls refused for pipe models and the custom params a control would set, Stream Chat Response off per chat and per model in a tool turn; see [Open WebUI 0.12 features](#open-webui-012-features-geminiowui012)) | | `azure` | `pipelines/azure/azure_ai_foundry.py` | `mock_azure.py` (requires `api-version`; answers prompts with a `` block like a plain model, without `context`; query generation, embeddings, tool calls, a ~300 KB and a > 4 MiB stream event, `content: null`), `mock_search.py` (Azure AI Search and an App Service managed identity token endpoint, see below) | `valves` (from 3.0.0 also the `AZURE_AI_SEARCH_*` valves, their defaults, enums and the password input of the key, and no `AZURE_AI_SEARCH_MODE`), `models` (`AZURE_AI_MODEL` lists separated by `;`, `,` or spaces with exact names, `AZURE_AI_PIPELINE_PREFIX`, model from an `*.openai.azure.com` URL, predefined and fallback models), `api` (api-key and Bearer header, path and api-version, allow-listed body: extra client keys dropped, tools forwarded, `stream_options` only for streams; JSON 400 (exactly Azure's message, nothing appended) and text/plain 500 errors), `dotted` (`gpt-4.1`, `Phi-3.5-mini-instruct` reach upstream intact, model in header or body), `browser` (full status sequence, error status with exactly Azure's message), `tasks` (title task, also with Azure AI Search valves, #123), `rag` (the pipe's own Azure AI Search retrieval, 3.0.0, #187; `suites/_azure_rag.py`: API and browser path, stream and non-stream, Foundry `/models` endpoint; search request body per `query_type`, query text rules, embeddings for `deployment_name` (v1 route, gateway prefix, Bearer) and `endpoint` (api-version, own key or access token, chat key only on the same scheme, host and port), integrated vectorizer; `fields_mapping` with `select`, separator and fallback, list-valued titles; `filter`, `top_n_documents` (2 x candidates), `strictness` (per query), the api-version valve, the document budget (auto, also with a large `top_n_documents`, valve, unlimited, dropping a document); prompt placement, one block and one system message, `in_scope` and the tool-results clause, `role_information`, sanitizing (also nested tags, titles and file names), list and image-only content, Open WebUI's tool-images message, an attached file (`` and RAG template around the prompt; the search text is the typed prompt from `__metadata__`: `base_user_prompt` of Open WebUI 0.12 or `user_prompt` of 0.11, both also checked with the staged file in the driver process, `query-text.metadata`); citations, `context` event and `message.context`, scores per score type and with `AZURE_AI_INCLUDE_SEARCH_SCORES` off; `[docX]` → links, also split across stream deltas (API and browser, also without a finish chunk; in API streams all text before `[DONE]` and nothing after it), already linked or with parentheses in the URL (stream and non-stream), with a dotted deployment name; links in the history sent back as `[docX]` (also links saved before 2.8.0 with `)` in the URL; a hostile history line in linear time); only referenced sources saved and the show-all valve (also for an answer that cites only a document the search did not return, `[doc9]`, or one next to `[doc1]`); a context event over 128 KiB, an upstream event of ~300 KB (read and passed on) and one over 4 MiB (an error, API and browser); `content: null`; tool rounds (one search, reused only by a tool round of the same message with unchanged valves, no duplicate sources also when the tool round references a document, a non-stream tool call), tools and `stream_options` forwarded, tasks without retrieval and without `data_sources` (also for a saved chat without a websocket session); query generation (follow-ups, `always` / `off`, transcript with Open WebUI's conversation summary, max queries, `` with draft queries, fallback also for JSON nested too deep, partial and all-failed results, merge order by reciprocal rank fusion and reranker score, model in the body, the pause after 3 timeouts in a row: per model, reset by a success, still on after 16 s); auth: key, key valve, access token, system- and user-assigned managed identity (`mi_res_id`), token errors; fail-closed errors with a terminal status (HTTP 500/403/402/404/400/302/429/503, connection, retries, the 45 s retrieval limit, an answer larger than 16 MB, configuration errors, client `data_sources` (API stream and non-stream and the browser path as from an inlet filter: the error naming the removal of On Your Data in 3.0.0, the pipe's ERROR line and the terminal error status, nothing forwarded; an empty list is ignored), `context_length_exceeded` (exactly Azure's message and the budget hint)); Stop during a search and during query generation; `AZURE_AI_SEARCH_MODE=on_your_data` stored by a 2.9.0 pre-release or set in the environment is ignored; `log.debug`: the staged file run in the driver process with every logger at DEBUG logs no key or token, see below; `notice.none`: no On Your Data retirement notice of 2.8.1 in the server log), `logs` (no API key, no search key or token, no citation text, no generated queries, document titles or URLs in the server log at INFO and above) | | `n8n` | `pipelines/n8n/n8n.py` | `mock_n8n.py` | `api` (request payload contract, bearer / Cloudflare headers, usage, `intermediateSteps` tool display with verbosity and truncation, `` blocks, history / `INPUT_FIELD` / `RESPONSE_FIELD` valves, plain text, NDJSON, SSE streams in separate and coalesced writes with plain lines and `event:` / `id:` / `retry:` fields, OpenAI-style chunks, UTF-8 characters split across writes, braces inside strings, a large object trickling in as small writes (server CPU), webhook error), `browser` (saved answer, usage and final status for JSON, NDJSON, UTF-8, an n8n error chunk, a broken stream and a webhook error; chat context sent to the workflow for chat turns vs. background tasks; Stop during a stream and a non-stream request), `tasks` (title task without and with a chat id) | | `infomaniak` | `pipelines/infomaniak/infomaniak.py` | `mock_infomaniak.py` | `models` (llm models only, `NAME_PREFIX`), `api` (product id and bearer key, allow-listed body, SSE stream normal, coalesced into one write and split mid-JSON; OpenAI-style and Infomaniak `error.description` errors with one log line each), `browser` (saved answer, usage and status events for those streams plus no final newline, CRLF, a broken stream and an upstream error; Stop during the stream and while waiting for the response headers), `tasks` | @@ -214,24 +214,27 @@ Checks per suite in strict known mode on Open WebUI **v0.12.0-slim**, **v0.11.4- and **v0.11.3-slim** (observed 2026-10-11 with `google_gemini.py` 1.20.0 and the `gemini.shared` and `gemini.owui012` groups, `azure_ai_foundry.py` 3.0.1, Time Token Tracker 2.7.0, `infomaniak.py` 2.2.2; before the two groups 549 checks on every version, -with `google_gemini.py` 1.19.1): +with `google_gemini.py` 1.19.1; with the thinking level checks +`thinking.level-models` and `.params-robust` and the 4 checks of the security review +fixes of 1.20.0, which run on 0.12 only): | Suite | Checks on v0.12.0 | Checks on v0.11.4 / v0.11.3 | PASS / KNOWN | | --- | ---: | ---: | ---: | -| `gemini` | 191 | 180 | all / 0 | +| `gemini` | 197 | 182 | all / 0 | | `azure` | 199 | 199 | all / 0 | | `n8n` | 51 | 51 | all / 0 | | `infomaniak` | 32 | 32 | all / 0 | | `filters` | 101 | 101 | all / 0 | -| **all** | **574** | **563** | **all / 0** | +| **all** | **580** | **565** | **all / 0** | No FAIL and no obsolete marker on any of the three versions. The `gemini.shared` group -has 13 checks on 0.12 and two before (`shared.unsupported`, `shared.legacy`), see +has 17 checks on 0.12 and two before (`shared.unsupported`, `shared.legacy`), see [Shared chats](#shared-chats-geminishared); the 12 `gemini.owui012` checks run on every version, and on 0.11 a check of a 0.12 feature checks what 0.11 does instead (see [Open WebUI 0.12 features](#open-webui-012-features-geminiowui012)). `gemini.imgedit.admin` expects on 0.12 that Open WebUI refuses an admin's turn in another user's chat (HTTP -404), and on 0.11 that the pipe reads that chat. Before the +404), and on 0.11 that the pipe reads that chat: the user's generated image, not the +admin's file the user named in turn 1 (the user may not read it). Before the compatibility changes, `main` (31a4e7d) had 542 PASS and 5 FAIL on v0.12.0-slim: the `server-log` checks of azure, n8n and infomaniak (the module names), `imgedit.admin` and `azure.rag.query-text.attached`. The checks of the 1.19.0 thinking block @@ -412,16 +415,23 @@ of a group with a grant on X, a viewer D (chat Y shared "Clone only" with D, and folder of A shared read-only with D) and an outsider C. Chat X starts with A's turn on `gemini-3.1-flash-image` with the upload `upload1` (generated image `final`); B, A and G then write in it (the mock's `final-image-` trigger gives each turn its own image). +A second chat X2 of A, shared with B with "Allow replies", covers the fixes of the +security review of 1.20.0: B names A's private file (`saved`) in a message, A then edits +twice with A's own link to it, B plants 30 links to files A may not use, A edits again. A test-only global filter (`e2e_chat_redirect`, installed for the group) hands the pipe another chat when an API request's prompt carries `e2e-chat=/`: it sets `chat_id` and `user_message_id` of `__metadata__`, as if Open WebUI had called the pipe for that chat without its own check. So the pipe's own check is tested -also for the users Open WebUI turns away before the pipe. `e2e-chat-fault=` +also for the users Open WebUI turns away before the pipe. `e2e-link=` sets +`chat_id` and a new `message_id`, so the pipe emits and links its generated images as +for a message of that chat. `e2e-chat-fault=` and `e2e-file-fault=` make the pipe's next call of -`Chats.get_accessible_chat_by_id` / `has_access_to_file` for that id raise (an inert, -signature-preserving async wrapper stays installed in the server process, so the pipe's -feature detection still sees the helpers on a later load). +`Chats.get_accessible_chat_by_id` / `has_access_to_file` for that id raise, and +`e2e-chat-truthy=` the next call of `Chats.get_accessible_chat_by_id` return +`True`, a truthy value that is not the chat (an inert, signature-preserving async +wrapper stays installed in the server process, so the pipe's feature detection still +sees the helpers on a later load). | Check | What it shows | | --- | --- | @@ -434,18 +444,26 @@ feature detection still sees the helpers on a later load). | `shared.pipe-admin` | filter: the admin (read access to every chat in 0.12, no write access) gets none of X | | `shared.pipe-viewer` | filter: D (read access to Y through the Clone only share and the read-only folder) gets none of Y | | `shared.pipe-outsider` | filter: C gets none of X | +| `shared.pipe-truthy` | filter, `e2e-chat-truthy`: the chat check answers `True` instead of the chat for the admin: none of X is sent (only the chat itself allows) | | `shared.chat-fault` | the chat check raises for B: nothing of X is sent, the error is logged, the answer completes | | `shared.links` | B's API request with markdown links: A's upload of X is read; a file of the admin and a file of A outside X are not (logged twice, without their ids) | | `shared.file-fault` | the file check raises for B's link to A's upload: it is not read, logged without its id | | `shared.video` | Veo image-to-video: B's markdown link to A's upload of X is sent as `instances[0].image` | +| `shared.deputy` | X2: A's turn after B's message with a link to A's private file sends `upload1`, `final` and `gen2`, not the private file (B may not read it; logged), which stays HTTP 404 for B; A's own link to it is sent in A's turn and, from A's message, in A's next turn | +| `shared.history-cap` | X2: after B's 30 planted links (the admin's file and A's private file) A's turn looks at no more than 20 images of earlier messages (4 x `IMAGE_HISTORY_MAX_REFERENCES`): at most 20 refusals logged, only B's newest generated image sent, the answer completes | +| `shared.link-chat` | filter, `e2e-link` to X2: B's generated image is linked to X2 (A may read it), C's two images are not (HTTP 404 for A; `Not linking generated files` logged twice), once decided by the image history (`e2e-chat=`) and once by the linking itself | On Open WebUI before 0.12 the group records `shared.unsupported` (PASS, `feature not in Open WebUI 0.11.4`) and `shared.legacy`: A's chat shared with B through a 0.11 share link with a user grant, Open WebUI lets B read the upload and the generated image, but the pipe keeps its owner / admin rule and does not read B's markdown link to the upload. -With `google_gemini.py` 1.19.1 on v0.12.0-slim, 10 of the 13 checks FAIL; `refused`, +With `google_gemini.py` 1.19.1 on v0.12.0-slim, 10 of the first 13 checks FAIL; `refused`, `pipe-viewer` and `pipe-outsider` pass (1.19.1 allowed only the owner or an admin). -The group takes about 35 s on 0.12 and a few seconds on 0.11. +With 1.20.0 before the security review fixes (3443671) the four checks of the fixes +FAIL and the other 13 pass: `pipe-truthy` (the admin got X's images), `deputy` (A's +turn sent `saved`), `history-cap` (15 refusals, but the private links were read as A's +own and filled the limit) and `link-chat` (A could read C's images, HTTP 200). +The group takes about one minute on 0.12 and a few seconds on 0.11. A mutation round on the access logic (2026-10-11, `--only 'gemini.(shared|imgedit)$'`) killed every mutant: the chat check always allowing (`pipe-admin`, `pipe-viewer`: the @@ -459,6 +477,13 @@ linking of generated images (`visible`, `participant`, `owner`, `group`, (13 `imgedit` checks and `legacy`) and `has_access_to_file` used without the 0.12 detection (`legacy`). The last two cannot differ on 0.12. +A second round on the fixes of the security review (2026-10-11, same `--only`, each +mutant through `--file`) killed every mutant as well: the author check off and the +author check using the requesting user (`deputy` and `history-cap` on v0.12.0-slim, +`imgedit.admin` on v0.11.4-slim), linking without the chat check (`link-chat`), no +cap on the images of earlier messages (`history-cap`: 30 refusals logged) and the chat +check back to `bool(chat)` (`pipe-truthy`). + ### Open WebUI 0.12 features (`gemini.owui012`) The group lives in `tests/e2e/suites/_gemini_owui012.py` (run by `suites/gemini.py` From 0e727907a4203a83ae42e8546af13be7116cde99 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:32:00 +0200 Subject: [PATCH 11/12] docs(testing): measured runtimes and meta-test of Gemini 1.20.0 +semver: none Co-Authored-By: Claude Opus 5.5 --- docs/testing.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 43db1d5..40867a7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -57,18 +57,20 @@ function versions: google_gemini.py 1.20.0, azure_ai_foundry.py 3.0.1, n8n.py 2. ... [PASS ] gemini.tools.builtin built-in tool, browser stream=True: Open WebUI runs it, ... ... ---- gemini: 398s +--- gemini: 415s ... === infomaniak === ... [PASS ] infomaniak.models.name-prefix NAME_PREFIX valve changes the model names, and changing it back restores them ... -SUMMARY: 561 PASS, 0 FAIL, 0 KNOWN in 1179s -total runtime: 1215s +SUMMARY: 565 PASS, 0 FAIL, 0 KNOWN in 1213s +total runtime: 1249s output: tests/e2e/out/20261009-111759-owui-e2e-111759-1234 ``` -A full run of all suites took about 20 minutes on a shared 8-CPU Docker host (TIMING). +A full run of all suites took about 20 minutes on a shared 8-CPU Docker host (2026-10-11, +`google_gemini.py` 1.20.0: 1213 s on `v0.11.4-slim`, 1263 s on `v0.12.0-slim`; up to 30 minutes +while other runs share the host). Most of it is waiting: the azure `rag` group adds about 5.5 minutes (query-generation timeouts, the query-generation pause and the 45 s retrieval limit are waited for; in the meta-test, where every mock answers HTTP 500, its requests fail at once), the gemini @@ -951,7 +953,7 @@ demand (*Actions → E2E → Run workflow*, with an image tag and a suites input | Job | What it does | | --- | --- | | `e2e` | `run.sh` with all suites in **strict known mode** (`E2E_STRICT_KNOWN=1`) against the default image, which is read from the `DEFAULT_IMAGE=` line of `run.sh` (the only place it is defined). The weekly run adds `ghcr.io/open-webui/open-webui:latest-slim`; a manual run uses the image tag input. Output directory as artifact, `summary.md` as job summary | -| `meta` | **Meta-test**: `main`'s function files (`--ref origin/main`) with every provider mock answering HTTP 500 (`E2E_MOCK_FAULT=500`), suites `gemini azure n8n infomaniak`. Nearly everything fails, and it must give **no KNOWN**: a KNOWN means the `evidence` of that known bug also matches an unrelated failure and would hide it. `REQUEST_SIDE_KNOWN` in the workflow may list bugs that can only show in the request the pipe sends upstream (they reproduce whatever the mock answers); it is empty, because the evidence of every registered bug also needs proof that the upstream answered (e.g. `HTTP 200` and the mock's answer). Observed 2026-10-09 (`main` 53b8495): 38 PASS / 197 FAIL / 0 KNOWN; on 2026-10-10 with `main` 3ff6cf9 (Azure pipeline 2.8.1) and the harness of the Azure pipeline 3.0.0 (azure `rag` group): 35 PASS / 327 FAIL / 0 KNOWN (466 s), and with `main` 4b80e24 (Azure pipeline 3.0.0, `google_gemini.py` 1.17.0) and the suites with the `gemini.tools` / `toolsapi` / `imgedit` groups: 57 PASS / 370 FAIL / 0 KNOWN (557 s); with `main` 2fc7bcc (`google_gemini.py` 1.18.0, `infomaniak.py` 2.2.2, no known bug registered) and the suites with the `owuitools` group and the checks of 1.19.0: 57 PASS / 386 FAIL / 0 KNOWN (545 s); on 2026-10-11 with `main` 31a4e7d (`google_gemini.py` 1.19.0, Azure pipeline 3.0.0) and the harness of the Open WebUI 0.12 changes (2 more checks): 57 PASS / 388 FAIL / 0 KNOWN (518 s); with `main` 6b99dcf (`google_gemini.py` 1.19.1) and the harness with the `gemini.shared` group (2 more checks on the default image): 58 PASS / 389 FAIL / 0 KNOWN (536 s); with `main` 6b99dcf (`google_gemini.py` 1.19.1) and the harness with the `gemini.owui012` group (12 more checks): 58 PASS / 399 FAIL / 0 KNOWN (527 s; `owui012.controls` passes, it needs no provider answer). The `filters` suite is left out because it uses no provider mock (its known bugs reproduce for real) | +| `meta` | **Meta-test**: `main`'s function files (`--ref origin/main`) with every provider mock answering HTTP 500 (`E2E_MOCK_FAULT=500`), suites `gemini azure n8n infomaniak`. Nearly everything fails, and it must give **no KNOWN**: a KNOWN means the `evidence` of that known bug also matches an unrelated failure and would hide it. `REQUEST_SIDE_KNOWN` in the workflow may list bugs that can only show in the request the pipe sends upstream (they reproduce whatever the mock answers); it is empty, because the evidence of every registered bug also needs proof that the upstream answered (e.g. `HTTP 200` and the mock's answer). Observed 2026-10-09 (`main` 53b8495): 38 PASS / 197 FAIL / 0 KNOWN; on 2026-10-10 with `main` 3ff6cf9 (Azure pipeline 2.8.1) and the harness of the Azure pipeline 3.0.0 (azure `rag` group): 35 PASS / 327 FAIL / 0 KNOWN (466 s), and with `main` 4b80e24 (Azure pipeline 3.0.0, `google_gemini.py` 1.17.0) and the suites with the `gemini.tools` / `toolsapi` / `imgedit` groups: 57 PASS / 370 FAIL / 0 KNOWN (557 s); with `main` 2fc7bcc (`google_gemini.py` 1.18.0, `infomaniak.py` 2.2.2, no known bug registered) and the suites with the `owuitools` group and the checks of 1.19.0: 57 PASS / 386 FAIL / 0 KNOWN (545 s); on 2026-10-11 with `main` 31a4e7d (`google_gemini.py` 1.19.0, Azure pipeline 3.0.0) and the harness of the Open WebUI 0.12 changes (2 more checks): 57 PASS / 388 FAIL / 0 KNOWN (518 s); with `main` 6b99dcf (`google_gemini.py` 1.19.1) and the harness with the `gemini.shared` group (2 more checks on the default image): 58 PASS / 389 FAIL / 0 KNOWN (536 s); with `main` 6b99dcf (`google_gemini.py` 1.19.1) and the harness with the `gemini.owui012` group (12 more checks): 58 PASS / 399 FAIL / 0 KNOWN (527 s; `owui012.controls` passes, it needs no provider answer); with `main` 6b99dcf and the harness of `google_gemini.py` 1.20.0 (`thinking.level-models`, `thinking.params-robust` and the shared-chat checks of the security review): 59 PASS / 402 FAIL / 0 KNOWN (595 s). The `filters` suite is left out because it uses no provider mock (its known bugs reproduce for real) | | `api` | `check_owui_api.sh latest` | `E2E_TIMEOUT` and the steps' `timeout-minutes` bound every job, so a hanging scenario From dd310381740635462bcd965d2302acbb8f86c2e4 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:11:41 +0200 Subject: [PATCH 12/12] test(e2e): API check knows submodule imports and getattr-checked methods check_owui_api.sh failed on google_gemini.py 1.20.0: "from open_webui.utils.access_control import files" imports a submodule, which the check looked for as a name in __init__.py. A submodule is now ok and the functions called on it (or its alias) are checked like methods. A missing method the function looks up with getattr first (feature detection, Chats.get_accessible_chat_by_id on 0.11.x) is a WARN. +semver: none Co-Authored-By: Claude Opus 5.5 --- docs/testing.md | 8 ++++-- tests/e2e/check_owui_api.sh | 51 ++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 40867a7..1b7d25b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -805,9 +805,13 @@ It reads every `from open_webui... import ...` and `import open_webui...` in `pipelines/` and `filters/`, fetches the corresponding Open WebUI backend modules at that tag (`gh api` when authenticated, otherwise `curl` from raw.githubusercontent.com) and checks that each imported module and symbol, each method called on it -(`Users.get_user_by_id`, ...) and each injected `__param__` still exists; definitions +(`Users.get_user_by_id`, ...) and each injected `__param__` still exists. An imported +name may also be a submodule (`from open_webui.utils.access_control import files`); then +the functions called on it (or on its `as` alias) are checked like methods. Definitions marked legacy/deprecated are reported as `WARN` (for example `SRC_LOG_LEVELS` is an empty -legacy dict since 0.10, so per-module log levels have no effect). It also prints the +legacy dict since 0.10, so per-module log levels have no effect), and so is a missing +method that the function looks up with `getattr(obj, "name")` first and works without +(`Chats.get_accessible_chat_by_id`, new in 0.12, is missing on 0.11.x). It also prints the frontmatter of every function, the event types Open WebUI persists and the versions of shared dependencies. A fetch that fails for another reason than "not found" is tried 3 times; if it still fails the result is `incomplete` (exit 2) rather than a missing diff --git a/tests/e2e/check_owui_api.sh b/tests/e2e/check_owui_api.sh index 6ca9b24..cb065b3 100755 --- a/tests/e2e/check_owui_api.sh +++ b/tests/e2e/check_owui_api.sh @@ -8,9 +8,12 @@ # modules at TAG (via `gh api` when authenticated, otherwise raw.githubusercontent.com # with curl) and reports # - every `from open_webui.X import a, b` symbol defined (or re-exported) in X; +# (or a submodule X.a, whose called functions are checked like methods); # definitions marked "legacy"/"deprecated" are flagged as WARN # - every module imported as `import open_webui.X [as y]` # - every method called on an imported class object (Users.get_user_by_id(...)) +# or function called on an imported submodule (files.has_access_to_file(...)); +# a missing one the function looks up with getattr(obj, "name") first is a WARN # - every __dunder__ parameter the functions declare still injected by Open WebUI # - frontmatter (version / required_open_webui_version / requirements) per file # - event types Open WebUI persists from __event_emitter__, versions of shared deps @@ -125,6 +128,7 @@ SYMBOLS=$(echo "$IMPORTS" | awk -F'\t' 'NF == 3 { echo echo "== open_webui imports" B='([^A-Za-z0-9_]|$)' +SUBMODULES="" # "from open_webui.X import submodule": "X.submodule" per line while IFS=$'\t' read -r mod name users; do [ -n "$name" ] || continue src=$(module_file "$mod") @@ -137,7 +141,10 @@ while IFS=$'\t' read -r mod name users; do continue fi def=$(grep -Em1 "^(async[[:space:]]+)?def[[:space:]]+${name}[[:space:]]*\(|^class[[:space:]]+$name$B|^[[:space:]]*${name}[[:space:]]*(:[^=]*)?=([^=]|$)|^[[:space:]]*(from[[:space:]].*)?import[[:space:]].*[[:space:],(]$name$B|^[[:space:]]+$name,?[[:space:]]*$" "$src" || true) - if [ -z "$def" ]; then + if [ -z "$def" ] && [ -s "$(module_file "$mod.$name")" ]; then + ok "$mod.$name (module)" + SUBMODULES="$SUBMODULES$mod.$name"$'\n' + elif [ -z "$def" ]; then bad "$mod.$name <- $users" elif echo "$def" | grep -qiE "legacy|deprecated"; then warn "$mod.$name is marked legacy/deprecated: $(echo "$def" | sed 's/^[[:space:]]*//' | cut -c1-90) <- $users" @@ -174,18 +181,38 @@ done <<<"$MODULES" echo echo "== methods called on imported objects" while IFS=$'\t' read -r mod name users; do - case "$name" in [A-Z][a-z]*) ;; *) continue ;; esac - src=$(module_file "$mod") + if echo "$SUBMODULES" | grep -qxF "$mod.$name"; then + # a submodule: its functions, called through the name or its alias + where=$mod.$name + modre=$(echo "$mod" | sed 's/[.]/[.]/g') + # shellcheck disable=SC2046 # the file list is split into words on purpose + objs=$(cd "$REPO" && cat $(echo "$users" | tr , ' ') | tr -d '\r' \ + | sed -nE "s/^[[:space:]]*from[[:space:]]+$modre[[:space:]]+import[[:space:]]+(.*[[:space:],(])?$name[[:space:]]+as[[:space:]]+([A-Za-z_][A-Za-z0-9_]*).*/\2/p" \ + | sort -u) + [ -n "$objs" ] || objs=$name + else + case "$name" in [A-Z][a-z]*) ;; *) continue ;; esac + where=$mod + objs=$name + fi + src=$(module_file "$where") [ -s "$src" ] || continue - # shellcheck disable=SC2046 # the file list is split into words on purpose - for method in $(cd "$REPO" && cat $(echo "$users" | tr , ' ') \ - | grep -oE "(^|[^A-Za-z0-9_.])$name\.[a-z_][a-z0-9_]*\(" \ - | sed -E "s/.*$name\.([a-z0-9_]+)\(/\1/" | sort -u); do - if grep -Eq "^[[:space:]]*(async[[:space:]]+)?def[[:space:]]+${method}[[:space:]]*\(" "$src"; then - ok "$name.$method() ($mod)" - else - bad "$name.$method() not found in $mod <- $users" - fi + for obj in $objs; do + # shellcheck disable=SC2046 # the file list is split into words on purpose + for method in $(cd "$REPO" && cat $(echo "$users" | tr , ' ') \ + | grep -oE "(^|[^A-Za-z0-9_.])$obj\.[a-z_][a-z0-9_]*\(" \ + | sed -E "s/.*$obj\.([a-z0-9_]+)\(/\1/" | sort -u); do + if grep -Eq "^[[:space:]]*(async[[:space:]]+)?def[[:space:]]+${method}[[:space:]]*\(" "$src"; then + ok "$obj.$method() ($where)" + # shellcheck disable=SC2046 # the file list is split into words on purpose + elif (cd "$REPO" && grep -qE "getattr\($obj,[[:space:]]*[\"']$method[\"']" \ + $(echo "$users" | tr , ' ')); then + # feature detection: the function checks for it and works without it + warn "$obj.$method() not in $where (optional, checked with getattr) <- $users" + else + bad "$obj.$method() not found in $where <- $users" + fi + done done done <<<"$SYMBOLS"