From e2e830a0b9fc1d0a99f6c7836e3ab6a15c2a7f7e Mon Sep 17 00:00:00 2001 From: defangdevs Date: Fri, 4 Sep 2026 22:18:40 -0700 Subject: [PATCH 1/2] Add an Azure preview stack, and the two fixes it uncovered The preview is an overlay rather than an edited copy of compose.yaml, so it stays in sync with production. extends selects only the services a preview should run, which is how discord-bot (live Discord token) is left out. Deploying it surfaced two app bugs against GPT-5.1 on Azure that Bedrock had never exposed: max_tokens is rejected in favour of max_completion_tokens, and Azure opens a stream with a chunk whose choices list is empty. --- .defang/preview | 14 ++++++++++++++ app/rag_system.py | 11 ++++++++++- compose.preview.yaml | 34 ++++++++++++++++++++++++++++++++++ compose.yaml | 8 ++++++-- 4 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 .defang/preview create mode 100644 compose.preview.yaml diff --git a/.defang/preview b/.defang/preview new file mode 100644 index 0000000..34a4acf --- /dev/null +++ b/.defang/preview @@ -0,0 +1,14 @@ +# Throwaway Azure stack for previewing changes. Deploy with: +# defang compose up -s preview -f compose.preview.yaml +# +# No AZURE_CLIENT_ID: that is the CI user-assigned identity. Run locally and the +# Azure CLI login is used instead. +# +# DEFANG_MODE=affordable, not balanced: balanced is gated by subscription tier +# ("this recipe is not available for your tier"), and a preview wants the cheap +# shape anyway. +AZURE_LOCATION="westus" +AZURE_SUBSCRIPTION_ID="f311c4db-e998-4c94-906c-7e2637303a05" +AZURE_TENANT_ID="12c0f515-fa47-402f-9982-de7646d3cb28" +DEFANG_PROVIDER=azure +DEFANG_MODE=affordable diff --git a/app/rag_system.py b/app/rag_system.py index f46f767..318d935 100644 --- a/app/rag_system.py +++ b/app/rag_system.py @@ -282,7 +282,10 @@ def answer_query_stream(self, query): model=os.getenv("MODEL"), messages=messages, temperature=0.25, - max_tokens=2048, + # GPT-5.x rejects max_tokens ("Unsupported parameter: 'max_tokens' is + # not supported with this model. Use 'max_completion_tokens' instead."). + # LiteLLM runs with --drop_params, so this stays safe on Bedrock too. + max_completion_tokens=2048, # Claude 4.5+ on Bedrock rejects temperature and top_p together # ("`temperature` and `top_p` cannot both be specified for this # model"), so send only temperature. top_p=1 was a no-op anyway. @@ -293,6 +296,12 @@ def answer_query_stream(self, query): for chunk in stream: try: logging.debug(f"Received chunk: {chunk}") + # Azure OpenAI opens the stream with a chunk carrying only + # prompt_filter_results and an empty "choices" list, and emits + # further choice-less chunks for content filtering. Indexing + # [0] on those raises IndexError and aborts the answer. + if not chunk["choices"]: + continue content = chunk["choices"][0]["delta"].get("content", "") collected_messages.append(content) yield content diff --git a/compose.preview.yaml b/compose.preview.yaml new file mode 100644 index 0000000..50c64e1 --- /dev/null +++ b/compose.preview.yaml @@ -0,0 +1,34 @@ +# Preview overlay: a throwaway Azure stack for testing changes before they reach +# production. Deploy with: +# +# defang compose up -s -f compose.preview.yaml +# +# It deliberately does NOT include compose.yaml wholesale. `extends` pulls in only +# the services a preview should run, which is how discord-bot is left out: it holds +# the live Discord token and a second copy would answer real users alongside +# production. Compose overlays can add and change keys but cannot delete a service, +# so selecting services here is the only way to drop one. +services: + app: + extends: + file: compose.yaml + service: app + # Never claim the production hostname. Azure's DNS writer only touches Azure + # zones, so it would silently skip this anyway, but a preview must not race + # production for the certificate either. Empty string clears the inherited value. + domainname: "" + + redis: + extends: + file: compose.yaml + service: redis + +models: + # On Azure the AI Foundry deployment is named after the SERVICE, while Defang + # wires the model ALIAS into MODEL — so they must match or every request fails + # with DeploymentNotFound. Once pulumi-defang#536 ships (the provider then reads + # the alias from the service's own --alias) this can become "chat-default", which + # is portable across clouds. The alias only picks chat vs embedding, so "llm" is + # a valid chat model either way. + llm: + model: llm diff --git a/compose.yaml b/compose.yaml index 9fbf1c4..8c0e791 100644 --- a/compose.yaml +++ b/compose.yaml @@ -36,8 +36,12 @@ services: test: ["CMD", "curl", "-f", "http://localhost:5050/"] interval: 30s timeout: 10s - retries: 5 - start_period: 240s + # Azure Container Apps caps a probe's InitialDelaySeconds at 60 and Defang maps + # start_period onto it, so 240s fails the deploy outright. Keep the startup + # budget inside that limit (60 + 10*30 = 360s); ECS treats start_period as a + # grace window, so this stays equivalent there. + retries: 10 + start_period: 60s depends_on: - redis models: From c5c6024ff11aeb2040e8a0f6628640d166d2daa5 Mon Sep 17 00:00:00 2001 From: defangdevs Date: Mon, 7 Sep 2026 14:53:31 -0700 Subject: [PATCH 2/2] Add DEFANG_TTL to the preview stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lio's review suggestion on PR #134: a forgotten preview stack should self-destruct rather than run forever. No effect yet — the feature needs a pulumi-defang release past PR 536, blocked on #414 (RELEASE_PAT) — but harmless to set now. --- .defang/preview | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.defang/preview b/.defang/preview index 34a4acf..699cdf2 100644 --- a/.defang/preview +++ b/.defang/preview @@ -7,8 +7,14 @@ # DEFANG_MODE=affordable, not balanced: balanced is gated by subscription tier # ("this recipe is not available for your tier"), and a preview wants the cheap # shape anyway. +# +# DEFANG_TTL so a forgotten preview self-destructs instead of running forever. +# No effect yet: the feature needs a pulumi-defang release past PR 536, and +# releases are blocked on DefangLabs/pulumi-defang#414 (RELEASE_PAT). Harmless +# to set now — it starts working the day that release ships. AZURE_LOCATION="westus" AZURE_SUBSCRIPTION_ID="f311c4db-e998-4c94-906c-7e2637303a05" AZURE_TENANT_ID="12c0f515-fa47-402f-9982-de7646d3cb28" DEFANG_PROVIDER=azure DEFANG_MODE=affordable +DEFANG_TTL=1d