diff --git a/.defang/production b/.defang/production new file mode 100644 index 0000000..09aa78f --- /dev/null +++ b/.defang/production @@ -0,0 +1,7 @@ +AZURE_LOCATION="westus" +AZURE_SUBSCRIPTION_ID="f311c4db-e998-4c94-906c-7e2637303a05" +AZURE_CLIENT_ID="6e15e174-fe70-4d0a-ae6a-f7f9ddb14a2a" +AZURE_TENANT_ID="12c0f515-fa47-402f-9982-de7646d3cb28" +DEFANG_PROVIDER=azure +DEFANG_MODE=balanced +DEFANG_WORKSPACE=DefangLabs diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index a0a4dcb..2f26d07 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -21,85 +21,6 @@ jobs: - name: Checkout Repo uses: actions/checkout@v5 - - name: Configure AWS Credentials for CI - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::488659951590:role/ci-role-d4fe904 # ciRoleArn from defang-io/infrastructure stack - - - name: Configure AWS Credentials for Corp Website Account - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-east-1 - role-chaining: true - role-duration-seconds: 3600 - role-to-assume: arn:aws:iam::407839483216:role/admin - - - name: Ensure Bedrock model access - run: | - set -uo pipefail - - # This step is a safety net, not a gate: it must never block a deploy. - # Every failure path below warns and exits 0. - - # The model lives under the top-level "models:" entry, e.g. - # models: - # llm: - # model: us.anthropic.claude-haiku-4-5-20251001-v1:0 - # "models:" and "model_var:" deliberately do not match this pattern. - profile=$(grep -oP '^\s*model:\s*\K[^\s"]+' compose.yaml | head -1) - if [ -z "$profile" ]; then - echo "::warning::no model found in compose.yaml - skipping Bedrock access check" - exit 0 - fi - - # Strip the cross-region inference-profile prefix ("us.", "eu.", ...) - # to get the bare foundation-model id the agreement is against. - model=$(echo "$profile" | sed -E 's/^[a-z]{2}\.//') - echo "compose model=$profile -> foundation model $model" - - # Anthropic is the one Bedrock provider that still needs an explicit - # per-account agreement; without it every request fails at RUNTIME with - # AccessDenied about aws-marketplace:Subscribe, because Bedrock tries to - # auto-subscribe using the ECS task role (which cannot, and should not). - case "$model" in - anthropic.*) ;; - *) echo "$model is not an Anthropic model - nothing to check"; exit 0 ;; - esac - - # The "us." profile can route to any of its regions, so cover all of them. - for region in us-east-1 us-east-2 us-west-2; do - status=$(aws bedrock get-foundation-model-availability --region "$region" \ - --model-id "$model" --query 'agreementAvailability.status' --output text 2>/dev/null) - if [ "$status" = "AVAILABLE" ]; then - echo "$region: already subscribed" - continue - fi - - echo "$region: no agreement ($status) - subscribing" - offer=$(aws bedrock list-foundation-model-agreement-offers --region "$region" \ - --model-id "$model" --query 'offers[0].offerToken' --output text 2>/dev/null) - if [ -z "$offer" ] || [ "$offer" = "None" ]; then - echo "::warning::$region: no agreement offer available for $model" - continue - fi - aws bedrock create-foundation-model-agreement --region "$region" \ - --model-id "$model" --offer-token "$offer" || { - echo "::warning::$region: could not subscribe to $model"; continue; } - - for _ in $(seq 1 12); do - status=$(aws bedrock get-foundation-model-availability --region "$region" \ - --model-id "$model" --query 'agreementAvailability.status' --output text 2>/dev/null) - if [ "$status" = "AVAILABLE" ]; then break; fi - sleep 10 - done - if [ "$status" = "AVAILABLE" ]; then - echo "$region: subscribed" - else - echo "::warning::$region: still $status after subscribing" - fi - done - - name: Set up cache for pip dependencies uses: actions/cache@v4 with: @@ -124,20 +45,31 @@ jobs: # expected its master key, so every request failed auth ("No connected db." # - LiteLLM falls back to a DB lookup for any key that is not the master). # `config rm` only warns when the config is already gone, so this is safe to - # re-run and can be dropped once it has run against every stack. + # re-run and can be dropped once it has run against every stack. Kept here + # (now against the new Azure project/stack) in case a stale AWS-era config + # value gets carried over by the CLI. - name: Remove the OPENAI_API_KEY config that shadows the LiteLLM master key uses: DefangLabs/defang-github-action@v2 with: cli-version: nightly - provider: aws + project: docs-chatbot + stack: production + provider: azure command: config rm OPENAI_API_KEY + # No cloud-credential step: the Defang CLI exchanges this job's GitHub Actions OIDC + # token for Azure credentials itself, using AZURE_CLIENT_ID / AZURE_TENANT_ID / + # AZURE_SUBSCRIPTION_ID from .defang/production. Requires a federated credential for + # subject repo:DefangLabs/docs-chatbot:environment:production on that client ID + # (added in defang-mvp's pulumi/infrastructure, see DefangLabs/docs-chatbot#126). - name: Deploy uses: DefangLabs/defang-github-action@v2 with: cli-version: nightly config-env-vars: ASK_TOKEN REBUILD_TOKEN SECRET_KEY SEGMENT_WRITE_KEY DISCORD_APP_ID DISCORD_TOKEN DISCORD_PUBLIC_KEY INTERCOM_TOKEN INTERCOM_ADMIN_ID - provider: aws + project: docs-chatbot + stack: production + provider: azure allow-upgrade: true env: diff --git a/app/rag_system.py b/app/rag_system.py index f46f767..318d935 100644 --- a/app/rag_system.py +++ b/app/rag_system.py @@ -282,7 +282,10 @@ def answer_query_stream(self, query): model=os.getenv("MODEL"), messages=messages, temperature=0.25, - max_tokens=2048, + # GPT-5.x rejects max_tokens ("Unsupported parameter: 'max_tokens' is + # not supported with this model. Use 'max_completion_tokens' instead."). + # LiteLLM runs with --drop_params, so this stays safe on Bedrock too. + max_completion_tokens=2048, # Claude 4.5+ on Bedrock rejects temperature and top_p together # ("`temperature` and `top_p` cannot both be specified for this # model"), so send only temperature. top_p=1 was a no-op anyway. @@ -293,6 +296,12 @@ def answer_query_stream(self, query): for chunk in stream: try: logging.debug(f"Received chunk: {chunk}") + # Azure OpenAI opens the stream with a chunk carrying only + # prompt_filter_results and an empty "choices" list, and emits + # further choice-less chunks for content filtering. Indexing + # [0] on those raises IndexError and aborts the answer. + if not chunk["choices"]: + continue content = chunk["choices"][0]["delta"].get("content", "") collected_messages.append(content) yield content diff --git a/compose.yaml b/compose.yaml index 9fbf1c4..3d05bf2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,6 @@ services: app: restart: always domainname: ask.defang.io - x-defang-dns-role: arn:aws:iam::258338292852:role/dnsadmin-39a19c3 platform: linux/amd64 build: context: ./app @@ -36,8 +35,12 @@ services: test: ["CMD", "curl", "-f", "http://localhost:5050/"] interval: 30s timeout: 10s - retries: 5 - start_period: 240s + # Azure Container Apps caps a probe's InitialDelaySeconds at 60 and Defang maps + # start_period onto it, so 240s fails the deploy outright. Keep the startup + # budget inside that limit (60 + 10*30 = 360s); ECS treats start_period as a + # grace window, so this stays equivalent there. + retries: 10 + start_period: 60s depends_on: - redis models: @@ -78,4 +81,8 @@ services: models: llm: - model: us.anthropic.claude-haiku-4-5-20251001-v1:0 + # On Azure, x-defang-llm provisions a managed AI Foundry deployment and picks + # the concrete model dynamically (no way to pin a specific model there) — + # the deployment is named after this alias, so it must be "llm" to match + # what MODEL gets wired to below, not a Bedrock model ID. + model: llm