From 8cf051b4d5f9e14e96673e3345f0129763e09c55 Mon Sep 17 00:00:00 2001 From: defangdevs Date: Fri, 4 Sep 2026 17:00:24 -0700 Subject: [PATCH 1/2] Migrate ask.defang.io deploy from AWS to Azure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - drop x-defang-dns-role (AWS-only; Azure's DNS writer only touches Azure DNS zones, same as defang-mvp PR 3187 for the website) - llm's x-defang-llm now provisions a managed Azure AI Foundry deployment instead of running the Bedrock gateway container, so MODEL becomes the deployment alias ("llm") instead of a Bedrock model ID — accepted per-Lio's call on DefangLabs/docs-chatbot#126 to let Azure pick the model - switch deploy.yaml from AWS role-chaining + Bedrock model-agreement steps to Azure OIDC (defang-github-action exchanges the workflow's own token, no credential step needed) and provider: azure - add .defang/production, reusing the shared UAMI already used by defang-mvp's website (needs a new federated credential for this repo's subject — see defang-mvp PR for that) No staging/dev deploy exists for this repo, so this takes effect the moment it merges to main. --- .defang/production | 7 +++ .github/workflows/deploy.yaml | 96 +++++------------------------------ compose.yaml | 7 ++- 3 files changed, 26 insertions(+), 84 deletions(-) create mode 100644 .defang/production diff --git a/.defang/production b/.defang/production new file mode 100644 index 0000000..09aa78f --- /dev/null +++ b/.defang/production @@ -0,0 +1,7 @@ +AZURE_LOCATION="westus" +AZURE_SUBSCRIPTION_ID="f311c4db-e998-4c94-906c-7e2637303a05" +AZURE_CLIENT_ID="6e15e174-fe70-4d0a-ae6a-f7f9ddb14a2a" +AZURE_TENANT_ID="12c0f515-fa47-402f-9982-de7646d3cb28" +DEFANG_PROVIDER=azure +DEFANG_MODE=balanced +DEFANG_WORKSPACE=DefangLabs diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index a0a4dcb..2f26d07 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -21,85 +21,6 @@ jobs: - name: Checkout Repo uses: actions/checkout@v5 - - name: Configure AWS Credentials for CI - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::488659951590:role/ci-role-d4fe904 # ciRoleArn from defang-io/infrastructure stack - - - name: Configure AWS Credentials for Corp Website Account - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-east-1 - role-chaining: true - role-duration-seconds: 3600 - role-to-assume: arn:aws:iam::407839483216:role/admin - - - name: Ensure Bedrock model access - run: | - set -uo pipefail - - # This step is a safety net, not a gate: it must never block a deploy. - # Every failure path below warns and exits 0. - - # The model lives under the top-level "models:" entry, e.g. - # models: - # llm: - # model: us.anthropic.claude-haiku-4-5-20251001-v1:0 - # "models:" and "model_var:" deliberately do not match this pattern. - profile=$(grep -oP '^\s*model:\s*\K[^\s"]+' compose.yaml | head -1) - if [ -z "$profile" ]; then - echo "::warning::no model found in compose.yaml - skipping Bedrock access check" - exit 0 - fi - - # Strip the cross-region inference-profile prefix ("us.", "eu.", ...) - # to get the bare foundation-model id the agreement is against. - model=$(echo "$profile" | sed -E 's/^[a-z]{2}\.//') - echo "compose model=$profile -> foundation model $model" - - # Anthropic is the one Bedrock provider that still needs an explicit - # per-account agreement; without it every request fails at RUNTIME with - # AccessDenied about aws-marketplace:Subscribe, because Bedrock tries to - # auto-subscribe using the ECS task role (which cannot, and should not). - case "$model" in - anthropic.*) ;; - *) echo "$model is not an Anthropic model - nothing to check"; exit 0 ;; - esac - - # The "us." profile can route to any of its regions, so cover all of them. - for region in us-east-1 us-east-2 us-west-2; do - status=$(aws bedrock get-foundation-model-availability --region "$region" \ - --model-id "$model" --query 'agreementAvailability.status' --output text 2>/dev/null) - if [ "$status" = "AVAILABLE" ]; then - echo "$region: already subscribed" - continue - fi - - echo "$region: no agreement ($status) - subscribing" - offer=$(aws bedrock list-foundation-model-agreement-offers --region "$region" \ - --model-id "$model" --query 'offers[0].offerToken' --output text 2>/dev/null) - if [ -z "$offer" ] || [ "$offer" = "None" ]; then - echo "::warning::$region: no agreement offer available for $model" - continue - fi - aws bedrock create-foundation-model-agreement --region "$region" \ - --model-id "$model" --offer-token "$offer" || { - echo "::warning::$region: could not subscribe to $model"; continue; } - - for _ in $(seq 1 12); do - status=$(aws bedrock get-foundation-model-availability --region "$region" \ - --model-id "$model" --query 'agreementAvailability.status' --output text 2>/dev/null) - if [ "$status" = "AVAILABLE" ]; then break; fi - sleep 10 - done - if [ "$status" = "AVAILABLE" ]; then - echo "$region: subscribed" - else - echo "::warning::$region: still $status after subscribing" - fi - done - - name: Set up cache for pip dependencies uses: actions/cache@v4 with: @@ -124,20 +45,31 @@ jobs: # expected its master key, so every request failed auth ("No connected db." # - LiteLLM falls back to a DB lookup for any key that is not the master). # `config rm` only warns when the config is already gone, so this is safe to - # re-run and can be dropped once it has run against every stack. + # re-run and can be dropped once it has run against every stack. Kept here + # (now against the new Azure project/stack) in case a stale AWS-era config + # value gets carried over by the CLI. - name: Remove the OPENAI_API_KEY config that shadows the LiteLLM master key uses: DefangLabs/defang-github-action@v2 with: cli-version: nightly - provider: aws + project: docs-chatbot + stack: production + provider: azure command: config rm OPENAI_API_KEY + # No cloud-credential step: the Defang CLI exchanges this job's GitHub Actions OIDC + # token for Azure credentials itself, using AZURE_CLIENT_ID / AZURE_TENANT_ID / + # AZURE_SUBSCRIPTION_ID from .defang/production. Requires a federated credential for + # subject repo:DefangLabs/docs-chatbot:environment:production on that client ID + # (added in defang-mvp's pulumi/infrastructure, see DefangLabs/docs-chatbot#126). - name: Deploy uses: DefangLabs/defang-github-action@v2 with: cli-version: nightly config-env-vars: ASK_TOKEN REBUILD_TOKEN SECRET_KEY SEGMENT_WRITE_KEY DISCORD_APP_ID DISCORD_TOKEN DISCORD_PUBLIC_KEY INTERCOM_TOKEN INTERCOM_ADMIN_ID - provider: aws + project: docs-chatbot + stack: production + provider: azure allow-upgrade: true env: diff --git a/compose.yaml b/compose.yaml index 9fbf1c4..27dbb50 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,6 @@ services: app: restart: always domainname: ask.defang.io - x-defang-dns-role: arn:aws:iam::258338292852:role/dnsadmin-39a19c3 platform: linux/amd64 build: context: ./app @@ -78,4 +77,8 @@ services: models: llm: - model: us.anthropic.claude-haiku-4-5-20251001-v1:0 + # On Azure, x-defang-llm provisions a managed AI Foundry deployment and picks + # the concrete model dynamically (no way to pin a specific model there) — + # the deployment is named after this alias, so it must be "llm" to match + # what MODEL gets wired to below, not a Bedrock model ID. + model: llm From 835814a31fdbc0e0569e439118a2c7f60a0c80de Mon Sep 17 00:00:00 2001 From: defangdevs Date: Mon, 7 Sep 2026 15:20:00 -0700 Subject: [PATCH 2/2] Fix Azure deploy: probe start_period cap and GPT-5.1 API differences - compose.yaml: Azure Container Apps caps a probe's InitialDelaySeconds at 60, and Defang maps start_period onto it, so the inherited 240s failed the deploy outright with ContainerAppProbeInitialDelaySecondsOutOfRange. Keep the startup budget inside that limit (60 + 10*30 = 360s); ECS treats start_period as a grace window, so this stays equivalent there. - app/rag_system.py: two real bugs against GPT-5.1 that Bedrock never exposed, found via a live Azure preview deploy: max_tokens is rejected in favor of max_completion_tokens, and Azure opens a stream with a chunk whose choices list is empty (content filtering), which raised IndexError and killed the answer mid-stream. Verified end to end against a throwaway Azure preview stack: Deploy succeeds, the llm service provisions via Azure AI Foundry with MODEL correctly wired to the "llm" deployment alias, and /v1/ask answers correctly on gpt-5.1. Stack has been torn down. Same fixes as DefangLabs/docs-chatbot#134, ported here so this PR is correct standalone regardless of merge order between the two. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01MPfV4G82XiSnHVUhyqXzXv --- app/rag_system.py | 11 ++++++++++- compose.yaml | 8 ++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/app/rag_system.py b/app/rag_system.py index f46f767..318d935 100644 --- a/app/rag_system.py +++ b/app/rag_system.py @@ -282,7 +282,10 @@ def answer_query_stream(self, query): model=os.getenv("MODEL"), messages=messages, temperature=0.25, - max_tokens=2048, + # GPT-5.x rejects max_tokens ("Unsupported parameter: 'max_tokens' is + # not supported with this model. Use 'max_completion_tokens' instead."). + # LiteLLM runs with --drop_params, so this stays safe on Bedrock too. + max_completion_tokens=2048, # Claude 4.5+ on Bedrock rejects temperature and top_p together # ("`temperature` and `top_p` cannot both be specified for this # model"), so send only temperature. top_p=1 was a no-op anyway. @@ -293,6 +296,12 @@ def answer_query_stream(self, query): for chunk in stream: try: logging.debug(f"Received chunk: {chunk}") + # Azure OpenAI opens the stream with a chunk carrying only + # prompt_filter_results and an empty "choices" list, and emits + # further choice-less chunks for content filtering. Indexing + # [0] on those raises IndexError and aborts the answer. + if not chunk["choices"]: + continue content = chunk["choices"][0]["delta"].get("content", "") collected_messages.append(content) yield content diff --git a/compose.yaml b/compose.yaml index 27dbb50..3d05bf2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -35,8 +35,12 @@ services: test: ["CMD", "curl", "-f", "http://localhost:5050/"] interval: 30s timeout: 10s - retries: 5 - start_period: 240s + # Azure Container Apps caps a probe's InitialDelaySeconds at 60 and Defang maps + # start_period onto it, so 240s fails the deploy outright. Keep the startup + # budget inside that limit (60 + 10*30 = 360s); ECS treats start_period as a + # grace window, so this stays equivalent there. + retries: 10 + start_period: 60s depends_on: - redis models: