diff --git a/deploy/compose/ai-gateway/.env.example b/deploy/compose/ai-gateway/.env.example index 566bae7..9b89967 100644 --- a/deploy/compose/ai-gateway/.env.example +++ b/deploy/compose/ai-gateway/.env.example @@ -11,6 +11,16 @@ TECHFLOW_FLARUM_API_KEY_SECRET_FILE=/protected/path/flarum-api-key TECHFLOW_COMMUNITY_INGEST_WEBHOOK_SECRET_FILE=/protected/path/community-ingest-webhook TECHFLOW_COMMUNITY_PUBLISH_ENABLED=false TECHFLOW_COMMUNITY_POLL_INTERVAL_SECONDS=60 +TECHFLOW_ARTIFACT_MAX_BYTES=1073741824 +TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES=10737418240 +TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES=107374182400 +TECHFLOW_COMMUNITY_ATTACHMENT_MAX_BYTES=1073741824 +TECHFLOW_COMMUNITY_ARCHIVE_MAX_BYTES=10737418240 +TECHFLOW_COMMUNITY_ATTACHMENT_TIMEOUT_SECONDS=7200 +TECHFLOW_COMMUNITY_ATTACHMENT_RETRIES=2 +TECHFLOW_ARTIFACT_MAINTENANCE_INTERVAL_SECONDS=900 +TECHFLOW_ARTIFACT_DISK_WARN_PERCENT=70 +TECHFLOW_ARTIFACT_DISK_CRITICAL_PERCENT=85 # Compose file secrets are bind-mounted. Keep the parent directory mode 700 and the files mode 644 # so the PostgreSQL init user and the non-root Gateway UID can read them without exposing the path. # Actual OpenAI mode is enabled only through an operator override that mounts a runtime-only diff --git a/deploy/compose/ai-gateway/compose.yml b/deploy/compose/ai-gateway/compose.yml index cac7546..a4d246a 100644 --- a/deploy/compose/ai-gateway/compose.yml +++ b/deploy/compose/ai-gateway/compose.yml @@ -124,8 +124,9 @@ services: TECHFLOW_SOURCE_MIRROR_ROOT: /var/lib/techflow-source-mirrors TECHFLOW_ARTIFACT_ROOT: /var/lib/techflow-artifacts TECHFLOW_ARTIFACT_RETENTION_HOURS: "24" - TECHFLOW_ARTIFACT_MAX_BYTES: "10485760" - TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES: "20971520" + TECHFLOW_ARTIFACT_MAX_BYTES: ${TECHFLOW_ARTIFACT_MAX_BYTES:-1073741824} + TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES: ${TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES:-10737418240} + TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES: ${TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES:-107374182400} TECHFLOW_ARTIFACT_MAX_ARCHIVE_ENTRIES: "100" TECHFLOW_ARTIFACT_MAX_COMPRESSION_RATIO: "20" TECHFLOW_ARTIFACT_MAX_LOG_EVIDENCE_CHARS: "120000" @@ -199,6 +200,11 @@ services: TECHFLOW_FLARUM_API_KEY_FILE: /run/secrets/flarum_api_key TECHFLOW_COMMUNITY_INGEST_WEBHOOK_FILE: /run/secrets/community_ingest_webhook TECHFLOW_COMMUNITY_POLL_INTERVAL_SECONDS: ${TECHFLOW_COMMUNITY_POLL_INTERVAL_SECONDS:-60} + TECHFLOW_COMMUNITY_ATTACHMENT_MAX_BYTES: ${TECHFLOW_COMMUNITY_ATTACHMENT_MAX_BYTES:-1073741824} + TECHFLOW_COMMUNITY_ARCHIVE_MAX_BYTES: ${TECHFLOW_COMMUNITY_ARCHIVE_MAX_BYTES:-10737418240} + TECHFLOW_COMMUNITY_ATTACHMENT_TIMEOUT_SECONDS: ${TECHFLOW_COMMUNITY_ATTACHMENT_TIMEOUT_SECONDS:-7200} + TECHFLOW_COMMUNITY_ATTACHMENT_RETRIES: ${TECHFLOW_COMMUNITY_ATTACHMENT_RETRIES:-2} + TECHFLOW_COMMUNITY_ATTACHMENT_TMP_DIR: /var/lib/techflow-community-poller/tmp TECHFLOW_GATEWAY_URL: http://gateway:8090 TECHFLOW_COMMUNITY_POLLER_STATE: /var/lib/techflow-community-poller/state.json command: ["python", "scripts/poll_flarum.py"] @@ -219,6 +225,30 @@ services: cap_drop: ["ALL"] security_opt: ["no-new-privileges:true"] + artifact-maintainer: + image: techflow/ai-gateway:${TECHFLOW_RAG_RELEASE:-issue-46} + restart: unless-stopped + environment: + TECHFLOW_ARTIFACT_ROOT: /var/lib/techflow-artifacts + TECHFLOW_ARTIFACT_RETENTION_HOURS: "24" + TECHFLOW_ARTIFACT_MAX_BYTES: ${TECHFLOW_ARTIFACT_MAX_BYTES:-1073741824} + TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES: ${TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES:-10737418240} + TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES: ${TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES:-107374182400} + TECHFLOW_ARTIFACT_MAX_ARCHIVE_ENTRIES: "100" + TECHFLOW_ARTIFACT_MAX_COMPRESSION_RATIO: "20" + TECHFLOW_ARTIFACT_MAX_LOG_EVIDENCE_CHARS: "120000" + TECHFLOW_ARTIFACT_MAINTENANCE_INTERVAL_SECONDS: ${TECHFLOW_ARTIFACT_MAINTENANCE_INTERVAL_SECONDS:-900} + TECHFLOW_ARTIFACT_DISK_WARN_PERCENT: ${TECHFLOW_ARTIFACT_DISK_WARN_PERCENT:-70} + TECHFLOW_ARTIFACT_DISK_CRITICAL_PERCENT: ${TECHFLOW_ARTIFACT_DISK_CRITICAL_PERCENT:-85} + command: ["python", "scripts/artifact_maintenance.py"] + volumes: + - techflow_artifacts:/var/lib/techflow-artifacts + read_only: true + tmpfs: + - /tmp:size=16m,mode=1777 + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + networks: rag_internal: internal: true diff --git a/deploy/compose/ai-gateway/scripts/set_large_upload_limits.py b/deploy/compose/ai-gateway/scripts/set_large_upload_limits.py new file mode 100644 index 0000000..7f9173a --- /dev/null +++ b/deploy/compose/ai-gateway/scripts/set_large_upload_limits.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Idempotently align an existing runtime .env with the Issue #72 upload limits.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + + +VALUES = { + "TECHFLOW_RAG_RELEASE": "issue-72-large-uploads-1g10g", + "TECHFLOW_ARTIFACT_MAX_BYTES": "1073741824", + "TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES": "10737418240", + "TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES": "107374182400", + "TECHFLOW_COMMUNITY_ATTACHMENT_MAX_BYTES": "1073741824", + "TECHFLOW_COMMUNITY_ARCHIVE_MAX_BYTES": "10737418240", + "TECHFLOW_COMMUNITY_ATTACHMENT_TIMEOUT_SECONDS": "7200", + "TECHFLOW_COMMUNITY_ATTACHMENT_RETRIES": "2", +} + + +def update(path: Path) -> None: + lines = path.read_text(encoding="utf-8").splitlines() + output: list[str] = [] + seen: set[str] = set() + for line in lines: + key = line.split("=", 1)[0].strip() if "=" in line and not line.lstrip().startswith("#") else "" + if key in VALUES: + output.append(f"{key}={VALUES[key]}") + seen.add(key) + else: + output.append(line) + output.extend(f"{key}={value}" for key, value in VALUES.items() if key not in seen) + temporary = path.with_name(path.name + ".issue72.tmp") + temporary.write_text("\n".join(output) + "\n", encoding="utf-8") + os.chmod(temporary, 0o600) + temporary.replace(path) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("env_file", type=Path) + args = parser.parse_args() + if not args.env_file.is_file(): + raise SystemExit("runtime env file does not exist") + update(args.env_file) + print("issue72_large_upload_limits=updated") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/flarum/issue72-large-upload-policy.sh b/deploy/flarum/issue72-large-upload-policy.sh new file mode 100755 index 0000000..6712a00 --- /dev/null +++ b/deploy/flarum/issue72-large-upload-policy.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +set -euo pipefail + +APP_DIR="${FLARUM_APP_DIR:-/var/www/html}" +APP_USER="${FLARUM_RUN_USER:-www-data}" +BACKUP_ROOT="${TECHFLOW_UPLOAD_BACKUP_ROOT:-/var/backups/techflow-flarum}" +PHP_OVERRIDE="/etc/php/8.3/fpm/conf.d/99-techflow-upload.ini" +NGINX_OVERRIDE="/etc/nginx/conf.d/techflow-upload.conf" +FLARUM_EXTEND="${APP_DIR}/extend.php" +POLICY_EXTENDER="${APP_DIR}/techflow-upload-policy.extend.php" +PHP_UPLOAD_TMP="/var/lib/flarum-upload-tmp" +NGINX_BODY_TMP="/var/lib/nginx/techflow-body" +REGULAR_MAX_BYTES=1073741824 +ARCHIVE_MAX_BYTES=10737418240 +MAX_KIB=10485760 + +nginx_site_file() { + if [[ -n ${TECHFLOW_NGINX_SITE_FILE:-} ]]; then + readlink -f "$TECHFLOW_NGINX_SITE_FILE" + return + fi + local candidate + candidate=$(grep -lR 'server_name[[:space:]].*community\.ablecloud\.io' /etc/nginx/sites-enabled 2>/dev/null | head -n1) + if [[ -z $candidate ]]; then + mapfile -t candidates < <(find /etc/nginx/sites-enabled -maxdepth 1 \( -type f -o -type l \) -print) + if [[ ${#candidates[@]} -eq 1 ]]; then + candidate=${candidates[0]} + fi + fi + [[ -n $candidate ]] || { echo "Community nginx site file not found" >&2; exit 2; } + readlink -f "$candidate" +} + +require_root() { + if [[ ${EUID} -ne 0 ]]; then + echo "root privileges are required" >&2 + exit 2 + fi +} + +settings_php() { + runuser -u "$APP_USER" -- php -r ' + require $argv[1]."/vendor/autoload.php"; + $config = include $argv[1]."/config.php"; + $db = $config["database"]; + $dsn = "mysql:host={$db["host"]};port=".($db["port"] ?? 3306).";dbname={$db["database"]};charset=utf8mb4"; + $pdo = new PDO($dsn, $db["username"], $db["password"], [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $prefix = $db["prefix"] ?? ""; + $mode = $argv[2]; + if ($mode === "export") { + $stmt = $pdo->prepare("SELECT `key`,value FROM {$prefix}settings WHERE `key` IN (?,?) ORDER BY `key`"); + $stmt->execute(["fof-upload.maxFileSize", "fof-upload.mimeTypes"]); + echo json_encode($stmt->fetchAll(PDO::FETCH_KEY_PAIR), JSON_UNESCAPED_SLASHES), PHP_EOL; + exit; + } + if ($mode === "policy") { + $values = [ + "fof-upload.maxFileSize" => "10485760", + "fof-upload.mimeTypes" => json_encode([ + "^image\\/(jpeg|png|gif|webp|avif|bmp|tiff|svg\\+xml)$" => ["adapter" => "local", "template" => "image-preview"], + "^text\\/(plain|csv)$" => ["adapter" => "local", "template" => "file"], + "^application\\/(zip|x-zip-compressed|gzip|x-gzip|pdf)$" => ["adapter" => "local", "template" => "file"], + ], JSON_UNESCAPED_SLASHES), + ]; + } else { + $values = json_decode(stream_get_contents(STDIN), true, 512, JSON_THROW_ON_ERROR); + } + $stmt = $pdo->prepare("INSERT INTO {$prefix}settings (`key`,value) VALUES (?,?) ON DUPLICATE KEY UPDATE value=VALUES(value)"); + $pdo->beginTransaction(); + foreach ($values as $key => $value) { $stmt->execute([$key, (string)$value]); } + $pdo->commit(); + ' "$APP_DIR" "$1" +} + +backup_file() { + local source=$1 target=$2 + if [[ -e "$source" ]]; then + cp -a "$source" "$target" + else + : > "${target}.absent" + fi +} + +restore_file() { + local source=$1 target=$2 + if [[ -e "${source}.absent" ]]; then + rm -f "$target" + else + install -o root -g root -m 0644 "$source" "$target" + fi +} + +verify() { + php-fpm8.3 -t >/dev/null + nginx -t >/dev/null + local settings + settings=$(settings_php export) + SETTINGS_JSON="$settings" python3 - <<'PY' +import json, os +values = json.loads(os.environ["SETTINGS_JSON"]) +assert values.get("fof-upload.maxFileSize") == "10485760", values +mime = json.loads(values["fof-upload.mimeTypes"]) +joined = " ".join(mime) +for expected in ("image", "text", "zip", "gzip"): + assert expected in joined, (expected, joined) +for blocked in ("7z", "rar", "iso", "bzip", "stuffit", "lha", "arj"): + assert blocked not in joined, (blocked, joined) +PY + local fpm_info + fpm_info=$(php-fpm8.3 -i 2>/dev/null) + grep -q '^upload_max_filesize => 10G => 10G' <<<"$fpm_info" + grep -q '^post_max_size => 11G => 11G' <<<"$fpm_info" + grep -q '^upload_tmp_dir => /var/lib/flarum-upload-tmp => /var/lib/flarum-upload-tmp' <<<"$fpm_info" + nginx -T 2>&1 | grep -q 'client_max_body_size 11G;' + php -l "$POLICY_EXTENDER" >/dev/null + grep -q 'techflow-upload-policy.extend.php' "$FLARUM_EXTEND" + runuser -u "$APP_USER" -- php -r ' + require $argv[1]."/vendor/autoload.php"; + require $argv[1]."/techflow-upload-policy.extend.php"; + assert(techflow_upload_limit("service.log", "text/plain") === 1073741824); + assert(techflow_upload_limit("support.zip", "application/zip") === 10737418240); + ' "$APP_DIR" + curl --fail --silent --show-error --max-time 20 \ + --header 'Host: community.ablecloud.io' \ + "${TECHFLOW_COMMUNITY_VERIFY_URL:-http://127.0.0.1/}" >/dev/null + echo "issue72_upload_policy=verified max_file_kib=${MAX_KIB}" +} + +apply_policy() { + require_root + install -d -o root -g root -m 0700 "$BACKUP_ROOT" + local stamp backup + stamp=$(date -u +%Y%m%dT%H%M%SZ) + backup="${BACKUP_ROOT}/issue72-${stamp}" + install -d -o root -g root -m 0700 "$backup" + settings_php export > "${backup}/settings.json" + chmod 0600 "${backup}/settings.json" + backup_file "$PHP_OVERRIDE" "${backup}/php-upload.ini" + backup_file "$NGINX_OVERRIDE" "${backup}/nginx-upload.conf" + backup_file "$FLARUM_EXTEND" "${backup}/extend.php" + backup_file "$POLICY_EXTENDER" "${backup}/techflow-upload-policy.extend.php" + local nginx_site + nginx_site=$(nginx_site_file) + backup_file "$nginx_site" "${backup}/nginx-site.conf" + printf '%s\n' "$nginx_site" > "${backup}/nginx-site.path" + + cat > "$PHP_OVERRIDE" <<'EOF' +upload_max_filesize=10G +post_max_size=11G +max_execution_time=7200 +max_input_time=7200 +memory_limit=512M +max_file_uploads=5 +upload_tmp_dir=/var/lib/flarum-upload-tmp +EOF + chmod 0644 "$PHP_OVERRIDE" + cat > "$NGINX_OVERRIDE" <<'EOF' +client_body_timeout 7200s; +fastcgi_read_timeout 7200s; +client_body_buffer_size 1m; +client_body_temp_path /var/lib/nginx/techflow-body 1 2; +EOF + chmod 0644 "$NGINX_OVERRIDE" + install -d -o "$APP_USER" -g "$APP_USER" -m 0700 "$PHP_UPLOAD_TMP" + install -d -o www-data -g www-data -m 0700 "$NGINX_BODY_TMP" + + NGINX_SITE="$nginx_site" python3 - <<'PY' +import os, pathlib, re +path = pathlib.Path(os.environ["NGINX_SITE"]) +text = path.read_text(encoding="utf-8") +if re.search(r"client_max_body_size\s+[^;]+;", text): + text = re.sub(r"client_max_body_size\s+[^;]+;", "client_max_body_size 11G;", text, count=1) +else: + text = re.sub(r"(server_name\s+[^;]+;)", r"\1\n client_max_body_size 11G;", text, count=1) +path.write_text(text, encoding="utf-8") +PY + + cat > "$POLICY_EXTENDER" <<'PHP' +listen(WillBeUploaded::class, function (WillBeUploaded $event): void { + $limit = techflow_upload_limit($event->uploadedFile->getClientOriginalName(), $event->mime); + $size = $event->uploadedFile->getSize(); + if ($size === false || $size < 1 || $size > $limit) { + $label = $limit === 10737418240 ? '압축파일은 10GiB 이하' : '일반 파일은 1GiB 이하'; + throw new ValidationException(['upload' => "{$label}만 업로드할 수 있습니다."]); + } + }), +]; +PHP + chown "$APP_USER:$APP_USER" "$POLICY_EXTENDER" + chmod 0644 "$POLICY_EXTENDER" + POLICY_EXTENDER="$POLICY_EXTENDER" FLARUM_EXTEND="$FLARUM_EXTEND" python3 - <<'PY' +import os, pathlib +path = pathlib.Path(os.environ["FLARUM_EXTEND"]) +text = path.read_text(encoding="utf-8") +marker = "techflow-upload-policy.extend.php" +if marker not in text: + before, token, after = text.partition("return [") + if not token: + raise SystemExit("Flarum extend.php return array not found") + text = before + "return array_merge(require __DIR__.'/" + marker + "', [" + after + index = text.rfind("];" ) + if index < 0: + raise SystemExit("Flarum extend.php closing array not found") + text = text[:index] + "]);" + text[index + 2:] + path.write_text(text, encoding="utf-8") +PY + chown "$APP_USER:$APP_USER" "$FLARUM_EXTEND" + + settings_php policy &2 + exit 2 + } + restore_file "${backup}/php-upload.ini" "$PHP_OVERRIDE" + restore_file "${backup}/nginx-upload.conf" "$NGINX_OVERRIDE" + restore_file "${backup}/extend.php" "$FLARUM_EXTEND" + restore_file "${backup}/techflow-upload-policy.extend.php" "$POLICY_EXTENDER" + local nginx_site + nginx_site=$(cat "${backup}/nginx-site.path") + restore_file "${backup}/nginx-site.conf" "$nginx_site" + settings_php import < "${backup}/settings.json" + (cd "$APP_DIR" && runuser -u "$APP_USER" -- php flarum cache:clear) + php-fpm8.3 -t >/dev/null + nginx -t >/dev/null + systemctl reload php8.3-fpm + systemctl reload nginx + echo "issue72_upload_policy=rolled_back backup=${backup}" +} + +case "${1:-}" in + apply) apply_policy ;; + verify) verify ;; + rollback) rollback "${2:-}" ;; + *) echo "usage: $0 {apply|verify|rollback BACKUP_DIR}" >&2; exit 2 ;; +esac diff --git a/docs/evidence/issue-72/large-upload-production-validation.json b/docs/evidence/issue-72/large-upload-production-validation.json new file mode 100644 index 0000000..aea220f --- /dev/null +++ b/docs/evidence/issue-72/large-upload-production-validation.json @@ -0,0 +1,91 @@ +{ + "schemaVersion": 2, + "issue": 72, + "validatedAt": "2026-08-16T10:23:12+09:00", + "units": { + "advertised": "GB", + "enforced": "GiB", + "regularBoundaryBytes": 1073741824, + "archiveBoundaryBytes": 10737418240 + }, + "environments": { + "flarum": { + "host": "172.16.0.234", + "flarum": "1.8.18", + "fofUpload": "1.8.5", + "rootSizeGiB": 1006, + "rootFreeGiB": 955, + "policyVerify": "passed" + }, + "techflow": { + "host": "172.16.0.231", + "release": "issue-72-large-uploads-1g10g", + "gatewayHealth": "healthy", + "rootFreeBytes": 983218327552 + } + }, + "policy": { + "regularMaxBytes": 1073741824, + "archiveMaxBytes": 10737418240, + "extractedMaxBytes": 107374182400, + "archiveEntriesMax": 100, + "compressionRatioMax": 20, + "downloadTimeoutSeconds": 7200, + "downloadRetries": 2, + "retentionHours": 24, + "maintenanceIntervalSeconds": 900, + "diskWarnPercent": 70, + "diskCriticalPercent": 85 + }, + "tests": { + "runtimeRegression": {"total": 263, "passed": 263}, + "flarumBoundary": [ + {"case": "regular exact 1 GiB", "bytes": 1073741824, "status": 200, "elapsedSeconds": 16, "cleanupStatus": 204, "result": "PASS"}, + {"case": "regular 1 GiB + 1 byte", "bytes": 1073741825, "status": 422, "residualRecords": 0, "elapsedSeconds": 12, "result": "PASS"}, + {"case": "ZIP exact 10 GiB", "bytes": 10737418240, "status": 200, "elapsedSeconds": 410, "cleanupStatus": 204, "result": "PASS"}, + {"case": "ZIP 10 GiB + 1 byte", "bytes": 10737418241, "status": 413, "residualRecords": 0, "elapsedSeconds": 227, "result": "PASS"} + ], + "gatewayBoundary": [ + {"case": "regular exact 1 GiB", "bytes": 1073741824, "status": 201, "elapsedSeconds": 27.751, "result": "PASS"}, + {"case": "regular 1 GiB + 1 byte declared", "bytes": 1073741825, "status": 400, "result": "PASS"}, + {"case": "ZIP exact 10 GiB", "bytes": 10737418240, "status": 201, "elapsedSeconds": 294.814, "result": "PASS"}, + {"case": "ZIP 10 GiB + 1 byte declared", "bytes": 10737418241, "status": 400, "result": "PASS"} + ], + "memory": { + "gatewayVmHwmKiB": 61732, + "gatewayVmHwmMiB": 60.3 + }, + "security": [ + {"case": "path traversal", "status": 400, "result": "PASS"}, + {"case": "nested archive", "status": 400, "result": "PASS"}, + {"case": "compression bomb", "status": 400, "result": "PASS"}, + {"case": "executable member", "status": 400, "result": "PASS"}, + {"case": "image MIME mismatch", "status": 400, "result": "PASS"} + ], + "cleanup": { + "flarumUploadRows": 0, + "flarumUploadFiles": 0, + "gatewayArtifactsDeleted": 2, + "testContainersDeleted": 6, + "testVolumesDeleted": 7, + "temporaryFilesDeleted": true + }, + "artifactMaintenance": { + "level": "ok", + "diskUsedPercent": 5, + "freeBytes": 983218327552 + }, + "protectedService": { + "service": "github-chat-v1", + "state": "frozen", + "guard": "passed", + "containerIdsUnchanged": true + } + }, + "rollback": { + "flarumBackup": "/var/backups/techflow-flarum/issue72-20260816T010617Z", + "techflowBackup": "/home/ablecloud/techflow-ai-gateway/backups/issue72-1g10g-predeploy-20260816T010000Z", + "stagingRollbackRehearsed": true, + "databaseMigration": false + } +} diff --git a/docs/reports/issue-72-community-large-upload-validation.md b/docs/reports/issue-72-community-large-upload-validation.md new file mode 100644 index 0000000..922341e --- /dev/null +++ b/docs/reports/issue-72-community-large-upload-validation.md @@ -0,0 +1,103 @@ +# Issue #72 Community 대용량 첨부 개선 완료 보고서 + +## 결론 + +Issue #72의 대용량 첨부 정책을 일반 파일 1 GiB 이하, 지원 압축 파일 10 GiB 이하로 확대하고 운영 Flarum과 TechFlow AI Gateway에 적용했다. 정확한 경계 크기의 실파일을 사용한 운영 시험에서 일반 파일 1 GiB와 ZIP 10 GiB는 수용됐고, 각각 1바이트 초과 파일은 거부됐다. + +대용량 파일은 Poller와 Gateway가 디스크 기반으로 스트리밍하며, AI에는 원본 전체가 아니라 업로드 시 생성한 비밀정보 제거·요약 근거만 전달한다. 10 GiB 압축파일을 실제로 Gateway에서 분석했을 때 프로세스 최대 상주 메모리는 약 60.3 MiB였다. 시험 첨부, Artifact, 컨테이너, 볼륨과 임시 파일은 모두 삭제했고 운영 DB 잔존은 0건이다. + +## 완료 범위 + +| 완료 조건 | 결과 | +|---|---| +| 일반 파일 최대 1 GiB | Flarum 200, Gateway 201 | +| 지원 압축 파일 최대 10 GiB | Flarum 200, Gateway 201 | +| 각 상한 +1바이트 거부 | Flarum 422/413, Gateway 400/400 | +| 디스크 기반 스트리밍 | Poller 임시 볼륨, Gateway `.part` 파일 | +| 압축 안전 정책 | 최대 해제 100 GiB, 100개 항목, 20배 압축비 | +| 자동 회귀 | 263/263 통과 | +| 운영 적용·롤백 자산 | 적용/검증/롤백 스크립트와 백업 확보 | +| 보호 서비스 불변 | `github-chat-v1 state=frozen guard=passed` | + +## 계층별 운영값 + +| 계층 | 적용 전 | 적용 후 | +|---|---:|---:| +| Nginx 요청 | 120 MiB | 11 GiB, 7,200초 | +| PHP-FPM 파일/요청 | 120/120 MiB | 10/11 GiB | +| PHP-FPM 시간/메모리 | 30/60초, 128 MiB | 7,200/7,200초, 512 MiB | +| FoF Upload | 50 MiB | 전역 10 GiB | +| Flarum 유형 정책 | 50 MiB | 일반 1 GiB / 압축 10 GiB | +| Poller | 50 MiB, 120초 | 일반 1 GiB / 압축 10 GiB, 7,200초 | +| Gateway 원본/해제 | 50/100 MiB | 일반 1 GiB / 압축 10 GiB / 해제 100 GiB | + +구현 판정은 1 GiB=`1,073,741,824`바이트, 10 GiB=`10,737,418,240`바이트를 사용한다. 사용자 안내에서는 이해하기 쉽게 1GB·10GB라고 표시할 수 있으나 경계 시험과 코드 상수는 이진 단위로 고정했다. + +## 구현 내용 + +- Poller는 Flarum 첨부를 1 MiB 단위로 전용 임시 볼륨에 내려받고 Gateway로 다시 스트리밍한다. +- Gateway는 요청을 `.part` 파일에 순차 기록하면서 SHA-256을 계산한다. 알려진 Content-Length가 상한을 넘으면 본문 전송 전에 거부한다. +- ZIP, GZIP, TAR.GZ는 메모리에 한 번에 펼치지 않고 순차 검사한다. 경로 이탈, 링크·특수 파일, 실행 파일, 중첩 압축과 압축 폭탄을 거부한다. +- 대용량 원본은 AI 질의 때 다시 파싱하지 않는다. 업로드 때 만든 정규화 근거 파일의 해시를 확인한 뒤 필요한 내용만 전달한다. +- 이미지 입력의 기존 크기·해상도 정책은 유지한다. 일반 파일 1 GiB 상한이 이미지 디코딩 상한을 확대하지 않는다. +- Flarum 배포 스크립트는 실제 Nginx `server_name`이 운영 도메인과 다르더라도 단일 활성 사이트인 경우 안전하게 해당 사이트를 선택한다. + +## 시험 결과 + +### 운영 Flarum 실파일 경계 + +| 시험 | 바이트 | HTTP | 경과 | 저장 결과 | +|---|---:|---:|---:|---| +| 일반 파일 정확히 1 GiB | 1,073,741,824 | 200 | 16초 | 생성 후 204 삭제 | +| 일반 파일 1 GiB+1 | 1,073,741,825 | 422 | 12초 | 생성 0건 | +| ZIP 정확히 10 GiB | 10,737,418,240 | 200 | 410초 | 생성 후 204 삭제 | +| ZIP 10 GiB+1 | 10,737,418,241 | 413 | 227초 | 생성 0건 | + +Flarum DB의 시험 업로드 ID 150·151은 삭제 후 0건이며, `public/assets/files`에도 시험 파일이 남아 있지 않다. 업로드 임시 영역과 루트 파일시스템에는 955 GiB가 남아 있다. + +### Gateway 독립 경계 + +| 시험 | HTTP | 경과 | 판정 | +|---|---:|---:|---| +| 일반 파일 정확히 1 GiB | 201 | 27.751초 | PASS | +| 일반 파일 1 GiB+1 선언 | 400 | 선차단 | PASS | +| ZIP 정확히 10 GiB | 201 | 294.814초 | PASS | +| ZIP 10 GiB+1 선언 | 400 | 선차단 | PASS | + +성공 Artifact 두 건은 HTTP 200으로 삭제했다. 10 GiB 분석 중 프로세스 `VmHWM`은 61,732 KiB로 약 60.3 MiB였다. + +### 자동 회귀와 보안 + +PR #65 기반 런타임 오버레이에서 263건 전부 통과했다. 일반/압축 상한, 스트리밍 수신, Content-Length 선차단, 재시도, 외부 URL 차단, 압축 안전 정책, Community 대화와 기존 답변 품질 계약을 포함한다. + +경로 이탈 ZIP, 중첩 압축, 압축 폭탄, 실행 파일 포함 ZIP, 이미지 MIME 위장은 모두 HTTP 400으로 차단된다. + +## 운영 상태 + +- Flarum 1.8.18 / FoF Upload 1.8.5 / 업로드 정책 검증 통과 +- Gateway `techflow/ai-gateway:issue-72-large-uploads-1g10g`: healthy +- Community Poller: 반복 처리 `failed=0` +- Artifact Maintainer: `level=ok`, 디스크 사용 5%, 약 983.2 GB 여유 +- Flarum 루트: 1006 GiB 중 955 GiB 여유 +- GitHub→Chat 보호 서비스: 배포 전후 `frozen`, guard passed +- Activepieces app/worker/event-gateway/ingress/Redis/Postgres 컨테이너 ID 불변 + +초기 Poller는 새 Gateway가 준비되기 전에 두 번 `URLError`를 기록했지만, Gateway가 healthy가 된 뒤 반복 처리에서 `failed=0`으로 정상화됐다. + +## 배포와 롤백 + +Flarum 정책 적용 전 백업은 `/var/backups/techflow-flarum/issue72-20260816T010617Z`에 있다. TechFlow 배포 전 백업은 `/home/ablecloud/techflow-ai-gateway/backups/issue72-1g10g-predeploy-20260816T010000Z`에 있으며 런타임 파일과 권한 0600의 환경 파일을 포함한다. + +TechFlow는 Gateway, Community Poller, Artifact Maintainer만 재생성했다. Activepieces와 GitHub→Chat 구성은 배포 대상에 포함하지 않았다. DB 스키마 변경은 없다. + +## 정리 결과 + +- Flarum 시험 첨부 2건 삭제, 초과 파일 저장 0건 +- Gateway 시험 Artifact 2건 삭제 +- TechFlow 경계 시험 컨테이너 6개와 볼륨 7개 삭제 +- 1 GiB/10 GiB 실파일과 임시 소스·작업 디렉터리 삭제 +- 운영 백업은 롤백 자산으로 유지 + +## 판정 + +일반 파일 1 GiB와 지원 압축 파일 10 GiB 요구사항, 초과 거부, 스트리밍 처리, 운영 배포, 보호 서비스 불변과 정리 기준을 모두 충족했다. 운영 판정은 **GO**다. diff --git a/docs/runbooks/community-large-uploads.md b/docs/runbooks/community-large-uploads.md new file mode 100644 index 0000000..0fc0343 --- /dev/null +++ b/docs/runbooks/community-large-uploads.md @@ -0,0 +1,140 @@ +# Community 대용량 첨부 운영 Runbook + +## 목적 + +Community 질문에 이미지, 일반 로그, ZIP, GZIP, TAR.GZ를 첨부했을 때 Flarum 수신부터 TechFlow AI 분석까지 같은 정책으로 처리한다. 일반 파일은 파일당 1 GiB 이하, 지원 압축 파일은 파일당 10 GiB 이하를 허용한다. + +```mermaid +flowchart LR + U["사용자"] --> N["Nginx - 요청 11 GiB"] + N --> P["PHP-FPM - 파일 10 GiB / 요청 11 GiB"] + P --> F["Flarum - 일반 1 GiB / 압축 10 GiB"] + F --> C["Poller - 디스크 임시 저장"] + C --> G["Gateway - 디스크 스트리밍 수신"] + G --> S["압축 스트리밍 검사 - 최대 해제 100 GiB"] + S --> E["정규화 근거만 AI에 전달"] +``` + +## 운영 경계 + +| 계층 | 기준 | 운영값 | +|---|---|---:| +| Nginx | 요청 본문 상한 | 11 GiB | +| Nginx | 본문/응답 대기 | 7,200초 | +| PHP-FPM | 파일/요청 | 10 GiB / 11 GiB | +| PHP-FPM | 실행/입력/메모리 | 7,200초 / 7,200초 / 512 MiB | +| FoF Upload | 전역 파일 상한 | 10 GiB (10,485,760 KiB) | +| TechFlow Flarum 정책 | 일반/압축 | 1 GiB / 10 GiB | +| Poller | 일반/압축/시간/재시도 | 1 GiB / 10 GiB / 7,200초 / 2회 | +| AI Gateway | 일반/압축 | 1 GiB / 10 GiB | +| 압축 안전검사 | 해제/항목/압축비 | 100 GiB / 100개 / 20배 | +| Artifact | 보관/점검 | 24시간 / 15분 | +| 디스크 | 경고/위험 | 70% / 85% | + +1 GiB는 `1,073,741,824`바이트, 10 GiB는 `10,737,418,240`바이트다. 문서와 사용자 안내에서 GB라고 부르더라도 구현과 시험 판정은 이 이진 경계를 사용한다. + +## 처리 원칙 + +- Poller는 첨부를 1 MiB 단위로 전용 볼륨에 내려받고 Gateway로 다시 스트리밍한다. 전체 파일을 `bytes`나 `bytearray`로 만들지 않는다. +- Gateway는 요청을 `.part` 파일에 순차 기록하며 SHA-256을 동시에 계산한다. Content-Length가 상한을 넘으면 본문을 받기 전에 거부하고, 길이를 알 수 없는 요청은 쓰는 중 상한에서 중단한다. +- ZIP/GZIP/TAR.GZ는 항목을 메모리에 펼치지 않고 순차 읽는다. 압축 해제 크기, 압축비, 항목 수, 경로 이탈, 링크·특수 파일, 실행 파일, 중첩 압축을 검사한다. +- AI 질의에는 원본 대용량 파일을 다시 읽히지 않는다. 업로드 때 생성한 비밀정보 제거·요약 근거 파일만 사용하며 해당 근거의 SHA-256을 다시 확인한다. +- 이미지 입력은 종전 이미지 크기·차원 정책을 유지한다. 1 GiB 일반 상한은 이미지 디코딩 상한을 확대한다는 의미가 아니다. + +## 허용 및 거부 + +허용 대상은 PNG/JPEG/WebP 이미지, UTF-8 텍스트 로그와 JSON/CSV/TSV, ZIP, GZIP, TAR.GZ/TGZ다. PDF는 Community 보관은 가능하지만 현재 TechFlow 로그 분석 대상은 아니다. + +다음 조건은 안전하게 거부하고 사용자에게 재첨부 안내를 제공한다. + +- 일반 파일 1 GiB 초과 또는 지원 압축 파일 10 GiB 초과 +- Community 외부 주소 +- ZIP/TAR 내부 경로 이탈, 절대 경로, 드라이브 경로 +- 심볼릭 링크, 파이프, 장치 등 특수 파일 +- 실행 파일 또는 중첩 압축 +- 압축 해제 100 GiB, 항목 100개, 압축비 20배 초과 +- UTF-8이 아닌 로그, 바이너리 로그, 이미지 MIME 위장 + +## 적용 + +Flarum 서버에서 배포 자산을 설치한 뒤 적용한다. + +```bash +sudo /usr/local/sbin/techflow-flarum-upload-policy apply +sudo /usr/local/sbin/techflow-flarum-upload-policy verify +``` + +TechFlow는 런타임 전용 `.env`에 같은 값을 설정하고 다음 세 서비스만 교체한다. + +```bash +docker compose -f compose.yml -f compose.openai.override.yml config --quiet +docker compose -f compose.yml -f compose.openai.override.yml up -d --no-deps \ + gateway community-poller artifact-maintainer +``` + +GitHub→Chat 보호 서비스는 배포 대상이 아니다. 배포 전후에 보호 검사를 실행하고 컨테이너 ID·이미지·시작 시각이 같아야 한다. + +```bash +cd /opt/ablestack-techflow/activepieces +sudo python3 scripts/protected_service_guard.py \ + --lock protected-services.json --env-file .env \ + --compose compose.yml --ingress ingress/Caddyfile +``` + +## 경계 검증 + +Gateway 전용 시험은 정확한 1 GiB 일반 로그와 정확한 10 GiB ZIP64 파일을 디스크에서 생성·전송한다. 성공 Artifact는 시험 종료 시 삭제하고 생성 파일도 기본적으로 제거한다. + +```bash +python scripts/verify_large_upload_boundaries.py \ + --base-url http://127.0.0.1:8090 \ + --workdir /var/tmp/techflow-issue72-boundary \ + --timeout 7200 +``` + +정상 판정은 다음과 같다. + +| 시험 | 기대 HTTP | +|---|---:| +| 일반 1 GiB | 201 | +| 일반 1 GiB + 1바이트 선언 | 400 | +| ZIP 10 GiB | 201 | +| ZIP 10 GiB + 1바이트 선언 | 400 | +| Flarum 일반 1 GiB | 200 | +| Flarum 일반 1 GiB + 1바이트 | 422 | +| Flarum ZIP 10 GiB | 200 | +| Flarum ZIP 10 GiB + 1바이트 | 413 | + +## 상태 확인 + +```bash +sudo /usr/local/sbin/techflow-flarum-upload-policy verify +docker inspect techflow-ai-gateway-gateway-1 --format '{{.State.Health.Status}}' +docker logs --tail 20 techflow-ai-gateway-community-poller-1 +docker logs --tail 20 techflow-ai-gateway-artifact-maintainer-1 +``` + +Gateway는 `healthy`, Poller와 Maintainer는 `running`, 유지관리 로그는 `level=ok`여야 한다. Poller 볼륨과 Artifact 볼륨에 `.part` 파일이 장시간 남아 있지 않아야 한다. + +## 용량 계획과 장애 처리 + +10 GiB 압축 파일 하나를 처리할 때 Community 원본, Poller 임시본, Gateway 원본이 잠시 공존할 수 있다. 최악의 정상 동시 점유량은 압축 해제 자료를 별도 저장하지 않는 조건에서도 약 30 GiB이므로, 동시 업로드 수와 24시간 보관량을 포함해 여유 공간을 잡는다. + +| 현상 | 확인 | 조치 | +|---|---|---| +| 413/422 | Nginx/PHP/FoF/TechFlow 정책 | 11G, 10G, 10,485,760 KiB 및 종류별 1/10 GiB 확인 | +| Poller fetch 경고 | 첨부 URL, 시간 초과, 임시 볼륨 | 외부 URL 차단과 7,200초/2회 재시도 확인 | +| 압축 안전 거부 | 항목 수, 경로, 해제 크기, 압축비 | 로그만 담아 다시 압축하도록 안내 | +| 디스크 warning/critical | `df -h`, Maintainer 로그, `.part` | 만료/고아 파일 정리 후 신규 대용량 첨부 일시 제한 | +| 처리 지연 | Gateway CPU, `VmHWM`, 업로드 경과 시간 | 동시 처리 수를 낮추고 큐 기반 비동기 분석을 후속 검토 | + +## 롤백 + +Flarum은 적용 시 출력된 백업 경로만 사용한다. + +```bash +sudo /usr/local/sbin/techflow-flarum-upload-policy rollback \ + /var/backups/techflow-flarum/issue72-YYYYMMDDTHHMMSSZ +``` + +TechFlow는 배포 전 런타임 파일과 `.env` 백업을 복원하고 이전 릴리스 이미지로 Gateway, Poller, Maintainer만 다시 만든다. DB 스키마 변경은 없다. diff --git a/output/issue-72-large-upload-artifact-manifest.json b/output/issue-72-large-upload-artifact-manifest.json new file mode 100644 index 0000000..3052a4f --- /dev/null +++ b/output/issue-72-large-upload-artifact-manifest.json @@ -0,0 +1,73 @@ +{ + "schemaVersion": "1.0", + "issue": 72, + "generatedAt": "2026-08-16T01:33:18.483764+00:00", + "artifactCount": 13, + "artifacts": [ + { + "path": "deploy/flarum/issue72-large-upload-policy.sh", + "bytes": 9822, + "sha256": "b8794b38c55722bf46e7297422d7c9d364dc5652aec9f5603723e151c415312a" + }, + { + "path": "docs/evidence/issue-72/large-upload-production-validation.json", + "bytes": 3347, + "sha256": "46319a0ecb7a6fc6b889db70c9d4817ac2d20ef3f848542aa88f579276f6efda" + }, + { + "path": "docs/runbooks/community-large-uploads.md", + "bytes": 6924, + "sha256": "486f95e17d2bd69b920d6b2af8334b4a90917984247aa1ce5500801c99f6e197" + }, + { + "path": "docs/reports/issue-72-community-large-upload-validation.md", + "bytes": 6428, + "sha256": "10302837c332eda773addf302de242d13720c0832ae2c7a21b534853f9192d55" + }, + { + "path": "output/pdf/techflow-issue-72-large-upload-report.pdf", + "bytes": 102255, + "sha256": "7ff48ce4887a042ecd62eda7070137ae541e864829ce17eafc84a00fc502a0ee" + }, + { + "path": "output/presentation/techflow-issue-72-large-upload.pptx", + "bytes": 28607, + "sha256": "1bc2299df1374c31ae3f8674ad28c19a0ee185c8766c86410e14c90815f2f4c8" + }, + { + "path": "output/pdf/techflow-issue-72-large-upload-presentation.pdf", + "bytes": 201443, + "sha256": "dbd9f7166b41c9b82bfde65a3bfd9fd4cbc1b9b8752fcbec7f56a44a9b398aa8" + }, + { + "path": "tools/artifacts/issue-72/README.md", + "bytes": 772, + "sha256": "c9adfcd5583d57749c23e0fab19ee738da9fb3ea82c2606b4197b89cf9cc4a42" + }, + { + "path": "tools/artifacts/issue-72/build_report.py", + "bytes": 9318, + "sha256": "39dabe021fe8fdd7b40dab63e07f72572f72ddea2f29940ffebaa9928c48cb3d" + }, + { + "path": "tools/artifacts/issue-72/build_presentation.mjs", + "bytes": 7036, + "sha256": "3223003b538e0691387b4e53848545c5a572eb24de14006acae2a130e0f46ff7" + }, + { + "path": "tools/artifacts/issue-72/build_presentation_pdf.py", + "bytes": 955, + "sha256": "c3f6ef0d6f859c393db8badac856b0d3007e6a374bc84d05ef84e1c23f0153a0" + }, + { + "path": "tools/artifacts/issue-72/build_manifest.py", + "bytes": 1392, + "sha256": "0835ae666c6e8814de5a8cb64d3870ac37e9d3238f06d0a2d5d9e6bb146b2baa" + }, + { + "path": "tools/artifacts/issue-72/validate_artifacts.py", + "bytes": 1750, + "sha256": "779c8effc1128f9346cd3f7e1e10f9e37c40485465d6643709e60740b8d9a1d0" + } + ] +} diff --git a/output/pdf/techflow-issue-72-large-upload-presentation.pdf b/output/pdf/techflow-issue-72-large-upload-presentation.pdf new file mode 100644 index 0000000..ba03c27 Binary files /dev/null and b/output/pdf/techflow-issue-72-large-upload-presentation.pdf differ diff --git a/output/pdf/techflow-issue-72-large-upload-report.pdf b/output/pdf/techflow-issue-72-large-upload-report.pdf new file mode 100644 index 0000000..6a7ea57 Binary files /dev/null and b/output/pdf/techflow-issue-72-large-upload-report.pdf differ diff --git a/output/presentation/techflow-issue-72-large-upload.pptx b/output/presentation/techflow-issue-72-large-upload.pptx new file mode 100644 index 0000000..efc612e Binary files /dev/null and b/output/presentation/techflow-issue-72-large-upload.pptx differ diff --git a/services/ai-gateway/app/artifacts.py b/services/ai-gateway/app/artifacts.py index bf31ee9..8386bf3 100644 --- a/services/ai-gateway/app/artifacts.py +++ b/services/ai-gateway/app/artifacts.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from dataclasses import dataclass from datetime import datetime, timedelta, timezone import hashlib @@ -10,9 +11,10 @@ from pathlib import Path import struct from threading import RLock +from typing import AsyncIterable, Iterable from uuid import UUID, uuid4 -from .log_artifacts import ARCHIVE_MEDIA_TYPES, PLAIN_MEDIA_TYPES, parse_log_artifact +from .log_artifacts import ARCHIVE_MEDIA_TYPES, PLAIN_MEDIA_TYPES, parse_log_artifact_path from .provider import EvidenceArtifact, ImageArtifact, LogArtifact from .store import InvalidBoundaryError, NotFoundError @@ -22,13 +24,13 @@ MEDIA_TYPE_ALIASES = {"application/x-zip-compressed", "application/octet-stream"} -def _normalized_media_type(filename: str, media_type: str, data: bytes) -> str: +def _normalized_media_type(filename: str, media_type: str, header: bytes) -> str: if media_type == "application/x-zip-compressed": return "application/zip" if media_type == "application/octet-stream": - if data[:4] in {b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"} and filename.casefold().endswith(".zip"): + if header[:4] in {b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"} and filename.casefold().endswith(".zip"): return "application/zip" - if data[:2] == b"\x1f\x8b" and filename.casefold().endswith((".gz", ".tgz")): + if header[:2] == b"\x1f\x8b" and filename.casefold().endswith((".gz", ".tgz")): return "application/gzip" return "text/plain" return media_type @@ -90,6 +92,7 @@ def payload(self) -> dict[str, object]: class ArtifactStore: def __init__( self, root: str, *, retention_hours: int, max_bytes: int, + max_archive_bytes: int = 10 * 1024 * 1024 * 1024, max_extracted_bytes: int = 20 * 1024 * 1024, max_archive_entries: int = 100, max_compression_ratio: int = 20, max_log_evidence_chars: int = 120_000, ) -> None: @@ -101,60 +104,142 @@ def __init__( pass self.retention = timedelta(hours=retention_hours) self.max_bytes = max_bytes + self.max_archive_bytes = max_archive_bytes self.max_extracted_bytes = max_extracted_bytes self.max_archive_entries = max_archive_entries self.max_compression_ratio = max_compression_ratio self.max_log_evidence_chars = max_log_evidence_chars self._lock = RLock() - def _paths(self, artifact_id: UUID) -> tuple[Path, Path]: + def _paths(self, artifact_id: UUID) -> tuple[Path, Path, Path]: base = self.root / str(artifact_id) - return base.with_suffix(".bin"), base.with_suffix(".json") + return base.with_suffix(".bin"), base.with_suffix(".json"), base.with_suffix(".evidence") - def put(self, filename: str, media_type: str, data: bytes) -> ArtifactRecord: - if media_type not in ALLOWED_MEDIA_TYPES | MEDIA_TYPE_ALIASES: - raise InvalidBoundaryError("unsupported evidence artifact media type") - if not data or len(data) > self.max_bytes: - raise InvalidBoundaryError("artifact size is outside the permitted boundary") + def _safe_name(self, filename: str) -> str: safe_name = Path(filename).name[:128] if not safe_name or safe_name != filename or "/" in filename or "\\" in filename: raise InvalidBoundaryError("artifact filename is invalid") - media_type = _normalized_media_type(safe_name, media_type, data) + return safe_name + + def _hint_limit(self, filename: str, media_type: str) -> int: + if media_type not in ALLOWED_MEDIA_TYPES | MEDIA_TYPE_ALIASES: + raise InvalidBoundaryError("unsupported evidence artifact media type") + lowered = filename.casefold() + if media_type in ARCHIVE_MEDIA_TYPES | {"application/x-zip-compressed"} or lowered.endswith((".zip", ".gz", ".tgz", ".tar.gz")): + return self.max_archive_bytes + return self.max_bytes + + @staticmethod + def _hash_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + def _finalize(self, temporary: Path, filename: str, media_type: str, size_bytes: int, sha256: str) -> ArtifactRecord: + if media_type not in ALLOWED_MEDIA_TYPES | MEDIA_TYPE_ALIASES: + raise InvalidBoundaryError("unsupported evidence artifact media type") + safe_name = self._safe_name(filename) + if not size_bytes: + raise InvalidBoundaryError("artifact size is outside the permitted boundary") + with temporary.open("rb") as source: + header = source.read(1024 * 1024) + media_type = _normalized_media_type(safe_name, media_type, header) + effective_max = self.max_archive_bytes if media_type in ARCHIVE_MEDIA_TYPES else self.max_bytes + if size_bytes > effective_max: + raise InvalidBoundaryError("artifact size is outside the permitted boundary") width = height = entry_count = extracted_bytes = None evidence_truncated = False redaction_count = 0 + evidence_text: str | None = None if media_type in IMAGE_MEDIA_TYPES: kind = "IMAGE" - width, height = _dimensions(data, media_type) + width, height = _dimensions(header, media_type) if width < 1 or height < 1 or width > 12000 or height > 12000 or width * height > 40_000_000: raise InvalidBoundaryError("artifact dimensions exceed the permitted boundary") else: kind = "LOG" - analysis = parse_log_artifact( - safe_name, media_type, data, max_entries=self.max_archive_entries, + analysis = parse_log_artifact_path( + safe_name, media_type, temporary, max_entries=self.max_archive_entries, max_extracted_bytes=self.max_extracted_bytes, max_ratio=self.max_compression_ratio, max_evidence_chars=self.max_log_evidence_chars, ) entry_count, extracted_bytes = analysis.entry_count, analysis.extracted_bytes evidence_truncated, redaction_count = analysis.truncated, analysis.redaction_count + evidence_text = analysis.evidence_text now, artifact_id = datetime.now(timezone.utc), uuid4() record = ArtifactRecord( - artifact_id, safe_name, media_type, hashlib.sha256(data).hexdigest(), len(data), kind, + artifact_id, safe_name, media_type, sha256, size_bytes, kind, width, height, entry_count, extracted_bytes, evidence_truncated, redaction_count, now, now + self.retention, ) - binary, metadata = self._paths(artifact_id) + binary, metadata, evidence = self._paths(artifact_id) + payload = record.payload() + if evidence_text is not None: + payload["evidenceSha256"] = hashlib.sha256(evidence_text.encode("utf-8")).hexdigest() with self._lock: - binary.write_bytes(data) - metadata.write_text(json.dumps(record.payload(), default=str, separators=(",", ":")), encoding="utf-8") + os.replace(temporary, binary) + if evidence_text is not None: + evidence.write_text(evidence_text, encoding="utf-8") + metadata.write_text(json.dumps(payload, default=str, separators=(",", ":")), encoding="utf-8") try: os.chmod(binary, 0o600); os.chmod(metadata, 0o600) + if evidence_text is not None: + os.chmod(evidence, 0o600) except OSError: pass return record + def _put_chunks(self, filename: str, media_type: str, chunks: Iterable[bytes]) -> ArtifactRecord: + safe_name = self._safe_name(filename) + hinted_max = self._hint_limit(safe_name, media_type) + temporary = self.root / f".upload-{uuid4().hex}.part" + digest, total = hashlib.sha256(), 0 + try: + with temporary.open("xb") as target: + for chunk in chunks: + if not chunk: + continue + total += len(chunk) + if total > hinted_max: + raise InvalidBoundaryError("artifact size is outside the permitted boundary") + digest.update(chunk) + target.write(chunk) + return self._finalize(temporary, safe_name, media_type, total, digest.hexdigest()) + finally: + temporary.unlink(missing_ok=True) + + def put(self, filename: str, media_type: str, data: bytes) -> ArtifactRecord: + return self._put_chunks(filename, media_type, (data,)) + + async def put_stream( + self, filename: str, media_type: str, chunks: AsyncIterable[bytes], *, content_length: int | None = None, + ) -> ArtifactRecord: + safe_name = self._safe_name(filename) + hinted_max = self._hint_limit(safe_name, media_type) + if content_length is not None and (content_length < 1 or content_length > hinted_max): + raise InvalidBoundaryError("artifact size is outside the permitted boundary") + temporary = self.root / f".upload-{uuid4().hex}.part" + digest, total = hashlib.sha256(), 0 + try: + with temporary.open("xb") as target: + async for chunk in chunks: + if not chunk: + continue + total += len(chunk) + if total > hinted_max: + raise InvalidBoundaryError("artifact size is outside the permitted boundary") + digest.update(chunk) + target.write(chunk) + return await asyncio.to_thread( + self._finalize, temporary, safe_name, media_type, total, digest.hexdigest() + ) + finally: + temporary.unlink(missing_ok=True) + def _load(self, artifact_id: UUID) -> ArtifactRecord: - binary, metadata = self._paths(artifact_id) + binary, metadata, _ = self._paths(artifact_id) if not binary.exists() or not metadata.exists(): raise NotFoundError("artifact not found") raw = json.loads(metadata.read_text(encoding="utf-8")) @@ -185,32 +270,39 @@ def image(self, artifact_id: UUID) -> ImageArtifact: def evidence(self, artifact_id: UUID) -> EvidenceArtifact: with self._lock: record = self._load(artifact_id) - binary, _ = self._paths(artifact_id) - data = binary.read_bytes() - if hashlib.sha256(data).hexdigest() != record.sha256: - raise InvalidBoundaryError("artifact integrity validation failed") + binary, metadata, evidence = self._paths(artifact_id) if record.kind == "IMAGE": + if self._hash_path(binary) != record.sha256: + raise InvalidBoundaryError("artifact integrity validation failed") + data = binary.read_bytes() return ImageArtifact(str(artifact_id), record.media_type, data, record.sha256) - analysis = parse_log_artifact( - record.filename, record.media_type, data, max_entries=self.max_archive_entries, - max_extracted_bytes=self.max_extracted_bytes, max_ratio=self.max_compression_ratio, - max_evidence_chars=self.max_log_evidence_chars, - ) - if ( - analysis.entry_count != record.entry_count or analysis.extracted_bytes != record.extracted_bytes - or analysis.redaction_count != record.redaction_count - ): - raise InvalidBoundaryError("artifact normalization integrity validation failed") + raw = json.loads(metadata.read_text(encoding="utf-8")) + if evidence.exists() and raw.get("evidenceSha256"): + evidence_text = evidence.read_text(encoding="utf-8") + if hashlib.sha256(evidence_text.encode("utf-8")).hexdigest() != raw["evidenceSha256"]: + raise InvalidBoundaryError("artifact normalization integrity validation failed") + else: + analysis = parse_log_artifact_path( + record.filename, record.media_type, binary, max_entries=self.max_archive_entries, + max_extracted_bytes=self.max_extracted_bytes, max_ratio=self.max_compression_ratio, + max_evidence_chars=self.max_log_evidence_chars, + ) + if ( + analysis.entry_count != record.entry_count or analysis.extracted_bytes != record.extracted_bytes + or analysis.redaction_count != record.redaction_count + ): + raise InvalidBoundaryError("artifact normalization integrity validation failed") + evidence_text = analysis.evidence_text return LogArtifact( - str(artifact_id), record.media_type, record.sha256, analysis.evidence_text, - analysis.entry_count, analysis.extracted_bytes, analysis.truncated, analysis.redaction_count, + str(artifact_id), record.media_type, record.sha256, evidence_text, + record.entry_count or 0, record.extracted_bytes or 0, record.evidence_truncated, record.redaction_count, ) def delete(self, artifact_id: UUID) -> bool: - binary, metadata = self._paths(artifact_id) - existed = binary.exists() or metadata.exists() + binary, metadata, evidence = self._paths(artifact_id) + existed = binary.exists() or metadata.exists() or evidence.exists() with self._lock: - binary.unlink(missing_ok=True); metadata.unlink(missing_ok=True) + binary.unlink(missing_ok=True); metadata.unlink(missing_ok=True); evidence.unlink(missing_ok=True) return existed def purge_expired(self) -> int: diff --git a/services/ai-gateway/app/config.py b/services/ai-gateway/app/config.py index 54a424c..ca63030 100644 --- a/services/ai-gateway/app/config.py +++ b/services/ai-gateway/app/config.py @@ -27,8 +27,9 @@ class Settings: database_pool_max: int = 4 artifact_root: str = os.path.join(tempfile.gettempdir(), "techflow-artifacts") artifact_retention_hours: int = 24 - artifact_max_bytes: int = 10 * 1024 * 1024 - artifact_max_extracted_bytes: int = 20 * 1024 * 1024 + artifact_max_bytes: int = 1024 * 1024 * 1024 + artifact_max_archive_bytes: int = 10 * 1024 * 1024 * 1024 + artifact_max_extracted_bytes: int = 100 * 1024 * 1024 * 1024 artifact_max_archive_entries: int = 100 artifact_max_compression_ratio: int = 20 artifact_max_log_evidence_chars: int = 120_000 @@ -54,8 +55,9 @@ def from_env(cls) -> "Settings": database_pool_max=int(os.getenv("TECHFLOW_RAG_DATABASE_POOL_MAX", "4")), artifact_root=os.getenv("TECHFLOW_ARTIFACT_ROOT", os.path.join(tempfile.gettempdir(), "techflow-artifacts")), artifact_retention_hours=int(os.getenv("TECHFLOW_ARTIFACT_RETENTION_HOURS", "24")), - artifact_max_bytes=int(os.getenv("TECHFLOW_ARTIFACT_MAX_BYTES", str(10 * 1024 * 1024))), - artifact_max_extracted_bytes=int(os.getenv("TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES", str(20 * 1024 * 1024))), + artifact_max_bytes=int(os.getenv("TECHFLOW_ARTIFACT_MAX_BYTES", str(1024 * 1024 * 1024))), + artifact_max_archive_bytes=int(os.getenv("TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES", str(10 * 1024 * 1024 * 1024))), + artifact_max_extracted_bytes=int(os.getenv("TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES", str(100 * 1024 * 1024 * 1024))), artifact_max_archive_entries=int(os.getenv("TECHFLOW_ARTIFACT_MAX_ARCHIVE_ENTRIES", "100")), artifact_max_compression_ratio=int(os.getenv("TECHFLOW_ARTIFACT_MAX_COMPRESSION_RATIO", "20")), artifact_max_log_evidence_chars=int(os.getenv("TECHFLOW_ARTIFACT_MAX_LOG_EVIDENCE_CHARS", "120000")), @@ -91,10 +93,12 @@ def validate(self) -> None: raise ConfigurationError("invalid database pool bounds") if not 1 <= self.artifact_retention_hours <= 168: raise ConfigurationError("TECHFLOW_ARTIFACT_RETENTION_HOURS must be between 1 and 168") - if not 1024 <= self.artifact_max_bytes <= 20 * 1024 * 1024: - raise ConfigurationError("TECHFLOW_ARTIFACT_MAX_BYTES must be between 1 KiB and 20 MiB") - if not self.artifact_max_bytes <= self.artifact_max_extracted_bytes <= 100 * 1024 * 1024: - raise ConfigurationError("TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES must be between upload max and 100 MiB") + if not 1024 <= self.artifact_max_bytes <= 1024 * 1024 * 1024: + raise ConfigurationError("TECHFLOW_ARTIFACT_MAX_BYTES must be between 1 KiB and 1 GiB") + if not self.artifact_max_bytes <= self.artifact_max_archive_bytes <= 10 * 1024 * 1024 * 1024: + raise ConfigurationError("TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES must be between regular max and 10 GiB") + if not self.artifact_max_archive_bytes <= self.artifact_max_extracted_bytes <= 100 * 1024 * 1024 * 1024: + raise ConfigurationError("TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES must be between archive max and 100 GiB") if not 1 <= self.artifact_max_archive_entries <= 500: raise ConfigurationError("TECHFLOW_ARTIFACT_MAX_ARCHIVE_ENTRIES must be between 1 and 500") if not 1 <= self.artifact_max_compression_ratio <= 100: @@ -115,7 +119,7 @@ def __repr__(self) -> str: "safety_identifier_salt_file=, embedding_batch_size={!r}, " "classification={!r}, log_level={!r}, " "database_pool_min={!r}, database_pool_max={!r}, artifact_root=, " - "artifact_retention_hours={!r}, artifact_max_bytes={!r}, artifact_max_extracted_bytes={!r}, " + "artifact_retention_hours={!r}, artifact_max_bytes={!r}, artifact_max_archive_bytes={!r}, artifact_max_extracted_bytes={!r}, " "artifact_max_archive_entries={!r}, artifact_max_compression_ratio={!r}, " "artifact_max_log_evidence_chars={!r}, flarum_base_url={!r}, flarum_public_url={!r}, " "flarum_api_key_file=, community_publish_enabled={!r})" @@ -130,6 +134,7 @@ def __repr__(self) -> str: self.database_pool_max, self.artifact_retention_hours, self.artifact_max_bytes, + self.artifact_max_archive_bytes, self.artifact_max_extracted_bytes, self.artifact_max_archive_entries, self.artifact_max_compression_ratio, diff --git a/services/ai-gateway/app/log_artifacts.py b/services/ai-gateway/app/log_artifacts.py index 857f94a..dbec15e 100644 --- a/services/ai-gateway/app/log_artifacts.py +++ b/services/ai-gateway/app/log_artifacts.py @@ -2,13 +2,16 @@ from __future__ import annotations +import codecs +from collections import deque from dataclasses import dataclass import gzip -from io import BytesIO from pathlib import Path, PurePosixPath import re import stat import tarfile +import tempfile +from typing import BinaryIO import zipfile from .store import InvalidBoundaryError @@ -24,6 +27,10 @@ INTERESTING = re.compile( r"(?i)(fatal|panic|exception|traceback|error|failed|failure|timeout|timed out|out of memory|oom|warn|denied|refused)" ) +INTERESTING_TOKENS = ( + b"fatal", b"panic", b"exception", b"traceback", b"error", b"failed", b"failure", b"timeout", + b"timed out", b"out of memory", b"oom", b"warn", b"denied", b"refused", +) SECRET_PATTERNS = ( re.compile(r"(?i)\b(authorization\s*:\s*(?:bearer|basic))\s+\S+"), re.compile(r"(?i)\b(password|passwd|pwd|secret|token|api[_-]?key)\b(\s*[:=]\s*)[^\s,;]+"), @@ -70,21 +77,6 @@ def _safe_member_name(name: str) -> str: return normalized -def _decode_log(data: bytes) -> str: - if not data: - raise InvalidBoundaryError("empty log entries are not permitted") - if b"\x00" in data: - raise InvalidBoundaryError("binary log content is not permitted") - try: - text = data.decode("utf-8-sig") - except UnicodeDecodeError as exc: - raise InvalidBoundaryError("log content must be UTF-8") from exc - controls = sum(ord(char) < 32 and char not in "\r\n\t" for char in text) - if controls > max(2, len(text) // 100): - raise InvalidBoundaryError("binary-like log content is not permitted") - return text.replace("\r\n", "\n").replace("\r", "\n") - - def _redact(text: str) -> tuple[str, int]: count = 0 for pattern in SECRET_PATTERNS: @@ -100,68 +92,205 @@ def replace(match: re.Match[str]) -> str: return text, count -def _selected_ranges(lines: list[str]) -> list[tuple[int, int]]: - interesting = {index for index, line in enumerate(lines) if INTERESTING.search(line)} - selected: set[int] = set() - if interesting: - for index in interesting: - selected.update(range(max(0, index - 2), min(len(lines), index + 3))) - else: - selected.update(range(min(20, len(lines)))) - selected.update(range(max(0, len(lines) - 20), len(lines))) - ranges: list[tuple[int, int]] = [] - for index in sorted(selected): - if not ranges or index > ranges[-1][1] + 1: - ranges.append((index, index)) +def parse_log_artifact( + filename: str, media_type: str, data: bytes, *, max_entries: int, max_extracted_bytes: int, + max_ratio: int, max_evidence_chars: int, +) -> LogAnalysis: + with tempfile.NamedTemporaryFile(prefix="techflow-log-", suffix=".bin") as temporary: + temporary.write(data) + temporary.flush() + return parse_log_artifact_path( + filename, media_type, Path(temporary.name), max_entries=max_entries, + max_extracted_bytes=max_extracted_bytes, max_ratio=max_ratio, + max_evidence_chars=max_evidence_chars, + ) + + +STREAM_CHUNK_BYTES = 1024 * 1024 +MAX_LOG_LINE_CHARS = 1024 * 1024 +CONTROL_BYTES = re.compile(rb"[\x01-\x08\x0b\x0c\x0e-\x1f]") + + +@dataclass(frozen=True) +class _EntryScan: + name: str + selected: tuple[tuple[int, str], ...] + bytes_read: int + truncated: bool + redaction_count: int + + +class _EvidenceSelector: + def __init__(self, name: str, max_chars: int) -> None: + self.name = name + self.max_chars = max_chars + self.first: list[tuple[int, str]] = [] + self.last: deque[tuple[int, str]] = deque(maxlen=20) + self.previous: deque[tuple[int, str]] = deque(maxlen=2) + self.selected: dict[int, str] = {} + self.selected_chars = 0 + self.interesting = False + self.after = 0 + self.line_number = 0 + self.redactions = 0 + self.truncated = False + + def _keep(self, number: int, line: str) -> None: + if number in self.selected: + return + rendered, count = _redact(line[:4000]) + if self.selected_chars + len(rendered) > self.max_chars * 2: + self.truncated = True + return + self.selected[number] = rendered + self.selected_chars += len(rendered) + self.redactions += count + + def line(self, raw: str) -> None: + self.line_number += 1 + line = raw.rstrip("\r\n") + item = (self.line_number, line) + if len(self.first) < 20: + self.first.append(item) + self.last.append(item) + if INTERESTING.search(line): + self.interesting = True + for number, previous in self.previous: + self._keep(number, previous) + self._keep(*item) + self.after = 2 + elif self.after: + self._keep(*item) + self.after -= 1 + self.previous.append(item) + + def plain_chunk(self, text: str) -> None: + """Advance a complete, non-interesting block without per-line regex calls.""" + line_count = text.count("\n") + if not line_count: + return + first_number = self.line_number + 1 + needed = max(0, 20 - len(self.first)) + if needed: + take = min(needed, line_count) + for offset, line in enumerate(text.split("\n", take)[:take]): + self.first.append((first_number + offset, line.rstrip("\r"))) + trailing = text[:-1].rsplit("\n", 20)[-20:] + trailing_start = first_number + line_count - len(trailing) + for offset, line in enumerate(trailing): + self.last.append((trailing_start + offset, line.rstrip("\r"))) + self.previous.clear() + self.previous.extend(self.last) + while len(self.previous) > 2: + self.previous.popleft() + self.line_number += line_count + + def result(self, bytes_read: int) -> _EntryScan: + if not self.line_number: + raise InvalidBoundaryError("empty log entries are not permitted") + if self.interesting: + selected = self.selected else: - ranges[-1] = (ranges[-1][0], index) - return ranges + selected = {} + for number, line in self.first + list(self.last): + if number not in selected: + redacted, count = _redact(line[:4000]) + selected[number] = redacted + self.redactions += count + return _EntryScan( + self.name, tuple(sorted(selected.items())), bytes_read, + self.truncated, self.redactions, + ) + + +def _scan_log_stream(name: str, stream: BinaryIO, *, max_bytes: int, max_evidence_chars: int) -> _EntryScan: + decoder = codecs.getincrementaldecoder("utf-8-sig")("strict") + selector = _EvidenceSelector(name, max_evidence_chars) + buffered = "" + total = controls = characters = 0 + try: + while True: + chunk = stream.read(STREAM_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise InvalidBoundaryError("archive expansion exceeds the permitted boundary") + if b"\x00" in chunk: + raise InvalidBoundaryError("binary log content is not permitted") + text = decoder.decode(chunk) + controls += len(CONTROL_BYTES.findall(chunk)) + characters += len(text) + combined = buffered + text + last_newline = combined.rfind("\n") + if last_newline < 0: + buffered = combined + else: + complete = combined[:last_newline + 1] + buffered = combined[last_newline + 1:] + lowered = complete.encode("utf-8").lower() + if selector.after or any(token in lowered for token in INTERESTING_TOKENS): + for line in complete[:-1].split("\n"): + selector.line(line) + else: + selector.plain_chunk(complete) + if len(buffered) > MAX_LOG_LINE_CHARS: + raise InvalidBoundaryError("log line exceeds the permitted boundary") + buffered += decoder.decode(b"", final=True) + except UnicodeDecodeError as exc: + raise InvalidBoundaryError("log content must be UTF-8") from exc + if buffered: + selector.line(buffered) + if controls > max(2, characters // 100): + raise InvalidBoundaryError("binary-like log content is not permitted") + return selector.result(total) -def _evidence(entries: list[tuple[str, str]], max_chars: int) -> tuple[str, bool, int]: +def _render_streamed_evidence(entries: list[_EntryScan], max_chars: int) -> tuple[str, bool, int]: blocks: list[str] = [] - redactions = 0 - truncated = False - for name, raw_text in entries: - text, count = _redact(raw_text) - redactions += count - lines = text.splitlines() - for start, end in _selected_ranges(lines): - rendered = [f"@@ {name}:{start + 1}-{end + 1}"] - rendered.extend(f"{number + 1}: {lines[number][:4000]}" for number in range(start, end + 1)) - block = "\n".join(rendered) + "\n" - if sum(len(item) for item in blocks) + len(block) > max_chars: + used = 0 + truncated = any(entry.truncated for entry in entries) + redactions = sum(entry.redaction_count for entry in entries) + for entry in entries: + selected = list(entry.selected) + index = 0 + while index < len(selected): + start = index + while index + 1 < len(selected) and selected[index + 1][0] == selected[index][0] + 1: + index += 1 + end = index + lines = selected[start:end + 1] + block = [f"@@ {entry.name}:{lines[0][0]}-{lines[-1][0]}"] + block.extend(f"{number}: {line}" for number, line in lines) + rendered = "\n".join(block) + "\n" + if used + len(rendered) > max_chars: truncated = True - remaining = max_chars - sum(len(item) for item in blocks) + remaining = max_chars - used if remaining > 128: - blocks.append(block[:remaining] + "\n[TRUNCATED]\n") + blocks.append(rendered[:remaining] + "\n[TRUNCATED]\n") return "".join(blocks), truncated, redactions - blocks.append(block) + blocks.append(rendered) + used += len(rendered) + index += 1 return "".join(blocks), truncated, redactions -def _validate_limits( - entries: list[tuple[str, bytes]], compressed_bytes: int, *, max_entries: int, - max_extracted_bytes: int, max_ratio: int, -) -> int: - if not entries or len(entries) > max_entries: - raise InvalidBoundaryError("archive entry count is outside the permitted boundary") - total = sum(len(data) for _, data in entries) - if total > max_extracted_bytes: - raise InvalidBoundaryError("extracted log size exceeds the permitted boundary") - if total > max(compressed_bytes, 1) * max_ratio: - raise InvalidBoundaryError("archive compression ratio exceeds the permitted boundary") - return total +def _archive_total_allowed(compressed_bytes: int, max_extracted_bytes: int, max_ratio: int) -> int: + return min(max_extracted_bytes, max(compressed_bytes, 1) * max_ratio) -def _read_zip(data: bytes, *, max_entries: int, max_extracted_bytes: int, max_ratio: int) -> list[tuple[str, bytes]]: +def _scan_zip_path( + path: Path, *, max_entries: int, max_extracted_bytes: int, max_ratio: int, max_evidence_chars: int, +) -> tuple[list[_EntryScan], int]: + compressed_bytes = path.stat().st_size + allowed = _archive_total_allowed(compressed_bytes, max_extracted_bytes, max_ratio) try: - with zipfile.ZipFile(BytesIO(data)) as archive: + with zipfile.ZipFile(path) as archive: infos = [item for item in archive.infolist() if not item.is_dir()] if not infos or len(infos) > max_entries: raise InvalidBoundaryError("archive entry count is outside the permitted boundary") - entries: list[tuple[str, bytes]] = [] declared_total = 0 + scans: list[_EntryScan] = [] for info in infos: name = _safe_member_name(info.filename) if info.flag_bits & 0x1: @@ -170,92 +299,117 @@ def _read_zip(data: bytes, *, max_entries: int, max_extracted_bytes: int, max_ra if file_type not in {0, stat.S_IFREG}: raise InvalidBoundaryError("archive links and special files are not permitted") declared_total += info.file_size - if declared_total > max_extracted_bytes or declared_total > max(len(data), 1) * max_ratio: + if declared_total > allowed: raise InvalidBoundaryError("archive expansion exceeds the permitted boundary") - member = archive.read(info) - if len(member) != info.file_size: + with archive.open(info) as stream: + scan = _scan_log_stream( + name, stream, max_bytes=info.file_size, max_evidence_chars=max_evidence_chars, + ) + if scan.bytes_read != info.file_size: raise InvalidBoundaryError("archive member size is inconsistent") - entries.append((name, member)) - _validate_limits(entries, len(data), max_entries=max_entries, max_extracted_bytes=max_extracted_bytes, max_ratio=max_ratio) - return entries + scans.append(scan) + return scans, declared_total except InvalidBoundaryError: raise - except (zipfile.BadZipFile, RuntimeError, EOFError) as exc: + except (zipfile.BadZipFile, RuntimeError, EOFError, OSError) as exc: raise InvalidBoundaryError("invalid ZIP log archive") from exc -def _read_gzip(data: bytes, filename: str, *, max_extracted_bytes: int, max_ratio: int) -> list[tuple[str, bytes]]: - try: - with gzip.GzipFile(fileobj=BytesIO(data)) as stream: - content = stream.read(max_extracted_bytes + 1) - except (gzip.BadGzipFile, EOFError, OSError) as exc: - raise InvalidBoundaryError("invalid GZIP log archive") from exc - if len(content) > max_extracted_bytes or len(content) > max(len(data), 1) * max_ratio: - raise InvalidBoundaryError("archive expansion exceeds the permitted boundary") +def _scan_gzip_path( + filename: str, path: Path, *, max_extracted_bytes: int, max_ratio: int, max_evidence_chars: int, +) -> tuple[list[_EntryScan], int]: name = filename[:-3] if filename.casefold().endswith(".gz") else filename + ".log" if not _is_log_name(name): raise InvalidBoundaryError("GZIP payload filename is not a supported log") - return [(Path(name).name, content)] + allowed = _archive_total_allowed(path.stat().st_size, max_extracted_bytes, max_ratio) + try: + with gzip.open(path, "rb") as stream: + scan = _scan_log_stream( + Path(name).name, stream, max_bytes=allowed, max_evidence_chars=max_evidence_chars, + ) + return [scan], scan.bytes_read + except InvalidBoundaryError: + raise + except (gzip.BadGzipFile, EOFError, OSError) as exc: + raise InvalidBoundaryError("invalid GZIP log archive") from exc -def _read_tar_gz( - data: bytes, *, max_entries: int, max_extracted_bytes: int, max_ratio: int, -) -> list[tuple[str, bytes]]: +def _scan_tar_gz_path( + path: Path, *, max_entries: int, max_extracted_bytes: int, max_ratio: int, max_evidence_chars: int, +) -> tuple[list[_EntryScan], int]: + allowed = _archive_total_allowed(path.stat().st_size, max_extracted_bytes, max_ratio) + scans: list[_EntryScan] = [] + total = 0 try: - with tarfile.open(fileobj=BytesIO(data), mode="r:gz") as archive: - members = [item for item in archive.getmembers() if not item.isdir()] - if not members or len(members) > max_entries: - raise InvalidBoundaryError("archive entry count is outside the permitted boundary") - entries: list[tuple[str, bytes]] = [] - declared_total = 0 - for member in members: + with tarfile.open(path, mode="r|gz") as archive: + for member in archive: + if member.isdir(): + continue if not member.isfile(): raise InvalidBoundaryError("archive links and special files are not permitted") + if len(scans) >= max_entries: + raise InvalidBoundaryError("archive entry count is outside the permitted boundary") name = _safe_member_name(member.name) - declared_total += member.size - if declared_total > max_extracted_bytes or declared_total > max(len(data), 1) * max_ratio: + total += member.size + if total > allowed: raise InvalidBoundaryError("archive expansion exceeds the permitted boundary") stream = archive.extractfile(member) if stream is None: raise InvalidBoundaryError("archive member cannot be read") - content = stream.read(member.size + 1) - if len(content) != member.size: + scan = _scan_log_stream( + name, stream, max_bytes=member.size, max_evidence_chars=max_evidence_chars, + ) + if scan.bytes_read != member.size: raise InvalidBoundaryError("archive member size is inconsistent") - entries.append((name, content)) - _validate_limits(entries, len(data), max_entries=max_entries, max_extracted_bytes=max_extracted_bytes, max_ratio=max_ratio) - return entries + scans.append(scan) + if not scans: + raise InvalidBoundaryError("archive entry count is outside the permitted boundary") + return scans, total + except InvalidBoundaryError: + raise except (tarfile.TarError, EOFError, OSError) as exc: raise InvalidBoundaryError("invalid TAR.GZ log archive") from exc -def parse_log_artifact( - filename: str, media_type: str, data: bytes, *, max_entries: int, max_extracted_bytes: int, +def parse_log_artifact_path( + filename: str, media_type: str, path: Path, *, max_entries: int, max_extracted_bytes: int, max_ratio: int, max_evidence_chars: int, ) -> LogAnalysis: lowered = filename.casefold() + with path.open("rb") as source: + header = source.read(4) if media_type in PLAIN_MEDIA_TYPES: if not _is_log_name(filename): raise InvalidBoundaryError("filename is not a supported log") - entries = [(filename, data)] + with path.open("rb") as source: + scans = [_scan_log_stream( + filename, source, max_bytes=path.stat().st_size, + max_evidence_chars=max_evidence_chars, + )] + extracted = scans[0].bytes_read elif media_type == "application/zip": - if not lowered.endswith(".zip") or data[:4] not in {b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"}: + if not lowered.endswith(".zip") or header not in {b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"}: raise InvalidBoundaryError("artifact bytes do not match ZIP") - entries = _read_zip(data, max_entries=max_entries, max_extracted_bytes=max_extracted_bytes, max_ratio=max_ratio) + scans, extracted = _scan_zip_path( + path, max_entries=max_entries, max_extracted_bytes=max_extracted_bytes, + max_ratio=max_ratio, max_evidence_chars=max_evidence_chars, + ) elif media_type in {"application/gzip", "application/x-gzip"}: - if not lowered.endswith((".gz", ".tgz")) or data[:2] != b"\x1f\x8b": + if not lowered.endswith((".gz", ".tgz")) or header[:2] != b"\x1f\x8b": raise InvalidBoundaryError("artifact bytes do not match GZIP") if lowered.endswith((".tar.gz", ".tgz")): - entries = _read_tar_gz(data, max_entries=max_entries, max_extracted_bytes=max_extracted_bytes, max_ratio=max_ratio) + scans, extracted = _scan_tar_gz_path( + path, max_entries=max_entries, max_extracted_bytes=max_extracted_bytes, + max_ratio=max_ratio, max_evidence_chars=max_evidence_chars, + ) else: - entries = _read_gzip(data, filename, max_extracted_bytes=max_extracted_bytes, max_ratio=max_ratio) + scans, extracted = _scan_gzip_path( + filename, path, max_extracted_bytes=max_extracted_bytes, + max_ratio=max_ratio, max_evidence_chars=max_evidence_chars, + ) else: raise InvalidBoundaryError("unsupported log artifact media type") - - extracted = _validate_limits( - entries, len(data), max_entries=max_entries, max_extracted_bytes=max_extracted_bytes, max_ratio=max_ratio, - ) - decoded = [(name, _decode_log(content)) for name, content in entries] - evidence, truncated, redactions = _evidence(decoded, max_evidence_chars) + evidence, truncated, redactions = _render_streamed_evidence(scans, max_evidence_chars) if not evidence: raise InvalidBoundaryError("log artifact produced no usable evidence") - return LogAnalysis(evidence, len(entries), extracted, truncated, redactions) + return LogAnalysis(evidence, len(scans), extracted, truncated, redactions) diff --git a/services/ai-gateway/app/main.py b/services/ai-gateway/app/main.py index 488f7ae..07de716 100644 --- a/services/ai-gateway/app/main.py +++ b/services/ai-gateway/app/main.py @@ -131,6 +131,7 @@ def create_app( runtime_settings.artifact_root, retention_hours=runtime_settings.artifact_retention_hours, max_bytes=runtime_settings.artifact_max_bytes, + max_archive_bytes=runtime_settings.artifact_max_archive_bytes, max_extracted_bytes=runtime_settings.artifact_max_extracted_bytes, max_archive_entries=runtime_settings.artifact_max_archive_entries, max_compression_ratio=runtime_settings.artifact_max_compression_ratio, @@ -447,7 +448,14 @@ async def create_artifact( if classification != "D0": raise InvalidBoundaryError("only D0 artifacts are permitted") media_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() - record = artifact_store.put(filename, media_type, await request.body()) + length_header = request.headers.get("Content-Length") + try: + content_length = int(length_header) if length_header is not None else None + except ValueError as exc: + raise InvalidBoundaryError("Content-Length must be an integer") from exc + record = await artifact_store.put_stream( + filename, media_type, request.stream(), content_length=content_length, + ) return _envelope(record.payload(), correlation_id) @application.get("/v1/artifacts/{artifactId}", response_model=Envelope, operation_id="getArtifact") diff --git a/services/ai-gateway/scripts/artifact_maintenance.py b/services/ai-gateway/scripts/artifact_maintenance.py new file mode 100644 index 0000000..661fc17 --- /dev/null +++ b/services/ai-gateway/scripts/artifact_maintenance.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Purge expired evidence artifacts and emit disk-capacity events.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import sys +import time + +sys.path.insert(0, str(Path(__file__).parents[1])) + +from app.artifacts import ArtifactStore + + +def _percent_env(name: str, default: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer") from exc + if not 1 <= value <= 99: + raise RuntimeError(f"{name} must be between 1 and 99") + return value + + +def maintain_once(root: Path) -> dict[str, int | str]: + store = ArtifactStore( + str(root), retention_hours=int(os.getenv("TECHFLOW_ARTIFACT_RETENTION_HOURS", "24")), + max_bytes=int(os.getenv("TECHFLOW_ARTIFACT_MAX_BYTES", str(1024 * 1024 * 1024))), + max_archive_bytes=int(os.getenv("TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES", str(10 * 1024 * 1024 * 1024))), + max_extracted_bytes=int(os.getenv("TECHFLOW_ARTIFACT_MAX_EXTRACTED_BYTES", str(100 * 1024 * 1024 * 1024))), + max_archive_entries=int(os.getenv("TECHFLOW_ARTIFACT_MAX_ARCHIVE_ENTRIES", "100")), + max_compression_ratio=int(os.getenv("TECHFLOW_ARTIFACT_MAX_COMPRESSION_RATIO", "20")), + max_log_evidence_chars=int(os.getenv("TECHFLOW_ARTIFACT_MAX_LOG_EVIDENCE_CHARS", "120000")), + ) + removed = store.purge_expired() + usage = shutil.disk_usage(root) + used_percent = round(usage.used * 100 / usage.total) if usage.total else 100 + warn_percent = _percent_env("TECHFLOW_ARTIFACT_DISK_WARN_PERCENT", 70) + critical_percent = _percent_env("TECHFLOW_ARTIFACT_DISK_CRITICAL_PERCENT", 85) + if warn_percent >= critical_percent: + raise RuntimeError("artifact disk warning threshold must be lower than critical threshold") + level = "critical" if used_percent >= critical_percent else ("warning" if used_percent >= warn_percent else "ok") + return { + "event": "artifact_maintenance_completed", "level": level, "removed": removed, + "usedPercent": used_percent, "freeBytes": usage.free, + } + + +def main() -> int: + root = Path(os.getenv("TECHFLOW_ARTIFACT_ROOT", "/var/lib/techflow-artifacts")) + interval = max(60, int(os.getenv("TECHFLOW_ARTIFACT_MAINTENANCE_INTERVAL_SECONDS", "900"))) + once = os.getenv("TECHFLOW_ARTIFACT_MAINTENANCE_ONCE", "false").lower() == "true" + while True: + print(json.dumps(maintain_once(root), separators=(",", ":")), flush=True) + if once: + return 0 + time.sleep(interval) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai-gateway/scripts/poll_flarum.py b/services/ai-gateway/scripts/poll_flarum.py index 1cb23c2..e0f34b7 100644 --- a/services/ai-gateway/scripts/poll_flarum.py +++ b/services/ai-gateway/scripts/poll_flarum.py @@ -5,14 +5,26 @@ from html.parser import HTMLParser import json +import mimetypes import os from pathlib import Path +import re +import tempfile import time import urllib.parse +import urllib.error import urllib.request from uuid import uuid4 +DEFAULT_ATTACHMENT_MAX_BYTES = 1024 * 1024 * 1024 +DEFAULT_ARCHIVE_MAX_BYTES = 10 * 1024 * 1024 * 1024 +DEFAULT_ATTACHMENT_TIMEOUT_SECONDS = 7200 +DEFAULT_ATTACHMENT_RETRIES = 2 +DOWNLOAD_CHUNK_BYTES = 1024 * 1024 +TRANSIENT_HTTP_STATUSES = {408, 425, 429, 500, 502, 503, 504} + + class ContentParser(HTMLParser): def __init__(self) -> None: super().__init__() @@ -45,6 +57,130 @@ def request_json(url: str, *, token: str | None = None, data: dict | None = None return json.loads(response.read().decode("utf-8")) +def _bounded_env_int(name: str, default: int, minimum: int, maximum: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer") from exc + if not minimum <= value <= maximum: + raise RuntimeError(f"{name} must be between {minimum} and {maximum}") + return value + + +def _attachment_policy() -> tuple[int, int, int, int]: + return ( + _bounded_env_int( + "TECHFLOW_COMMUNITY_ATTACHMENT_MAX_BYTES", DEFAULT_ATTACHMENT_MAX_BYTES, + 1024, DEFAULT_ATTACHMENT_MAX_BYTES, + ), + _bounded_env_int( + "TECHFLOW_COMMUNITY_ARCHIVE_MAX_BYTES", DEFAULT_ARCHIVE_MAX_BYTES, + DEFAULT_ATTACHMENT_MAX_BYTES, DEFAULT_ARCHIVE_MAX_BYTES, + ), + _bounded_env_int( + "TECHFLOW_COMMUNITY_ATTACHMENT_TIMEOUT_SECONDS", DEFAULT_ATTACHMENT_TIMEOUT_SECONDS, + 5, DEFAULT_ATTACHMENT_TIMEOUT_SECONDS, + ), + _bounded_env_int("TECHFLOW_COMMUNITY_ATTACHMENT_RETRIES", DEFAULT_ATTACHMENT_RETRIES, 0, 3), + ) + + +def _attachment_filename(content_disposition: str, path: str) -> str: + encoded = re.search(r"filename\*=UTF-8''([^;]+)", content_disposition, re.IGNORECASE) + quoted = re.search(r'filename="([^"]+)"', content_disposition, re.IGNORECASE) + value = urllib.parse.unquote(encoded.group(1)) if encoded else (quoted.group(1) if quoted else Path(path).name) + return Path(value.replace("\\", "/")).name[:128] or "community-artifact" + + +def _warning(filename: str, reason: str) -> str: + safe_name = Path(filename.replace("\\", "/")).name[:80] or "첨부파일" + messages = { + "size": f"첨부파일 {safe_name}이 허용 크기(일반 1GiB, 압축 10GiB)를 초과해 분석하지 않았습니다.", + "unsafe": f"첨부파일 {safe_name}은 지원하지 않거나 안전 검사를 통과하지 못해 분석에서 제외했습니다.", + "fetch": f"첨부파일 {safe_name}을 가져오지 못했습니다. 잠시 후 다시 첨부해 주세요.", + "origin": f"첨부파일 {safe_name}은 Community 외부 주소이므로 분석하지 않았습니다.", + } + return messages[reason] + + +def _normalized_attachment_media_type(filename: str, media_type: str) -> str: + if media_type not in {"application/force-download", "application/octet-stream"}: + return media_type + lowered = filename.casefold() + if lowered.endswith(".zip"): + return "application/zip" + if lowered.endswith((".tar.gz", ".tgz", ".gz")): + return "application/gzip" + if lowered.endswith((".log", ".txt", ".csv", ".ini")): + return "text/plain" + return mimetypes.guess_type(filename)[0] or media_type + + +def _is_archive(filename: str, media_type: str) -> bool: + normalized = _normalized_attachment_media_type(filename, media_type) + return normalized in {"application/zip", "application/gzip", "application/x-gzip"} + + +def _read_attachment( + request: urllib.request.Request, destination: Path, *, filename: str, + max_bytes: int, max_archive_bytes: int, timeout: int, retries: int, +) -> tuple[int, str, str, str]: + last_error: Exception | None = None + for attempt in range(retries + 1): + destination.unlink(missing_ok=True) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + media_type = response.headers.get_content_type() + disposition = response.headers.get("Content-Disposition") or "" + resolved_name = _attachment_filename(disposition, urllib.parse.urlparse(request.full_url).path) or filename + limit = max_archive_bytes if _is_archive(resolved_name, media_type) else max_bytes + content_length = response.headers.get("Content-Length") + if content_length and int(content_length) > limit: + raise ValueError("size") + total = 0 + with destination.open("xb") as target: + while True: + chunk = response.read(DOWNLOAD_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > limit: + raise ValueError("size") + target.write(chunk) + return total, media_type, disposition, resolved_name + except urllib.error.HTTPError as exc: + last_error = exc + if exc.code not in TRANSIENT_HTTP_STATUSES or attempt >= retries: + raise + except urllib.error.URLError as exc: + last_error = exc + if attempt >= retries: + raise + if attempt < retries: + time.sleep(min(2 ** attempt, 2)) + assert last_error is not None + raise last_error + + +def _file_chunks(path: Path): + with path.open("rb") as source: + while chunk := source.read(DOWNLOAD_CHUNK_BYTES): + yield chunk + + +def _upload_artifact( + gateway_url: str, path: Path, filename: str, media_type: str, correlation: str, timeout: int, +) -> str: + upload = urllib.request.Request( + gateway_url.rstrip("/") + "/v1/artifacts", data=_file_chunks(path), method="POST", + headers={"Content-Type": media_type, "Content-Length": str(path.stat().st_size), + "X-Artifact-Filename": filename, "X-Artifact-Classification": "D0", + "X-Correlation-Id": correlation}, + ) + with urllib.request.urlopen(upload, timeout=timeout) as response: + return str(json.loads(response.read().decode("utf-8"))["data"]["artifactId"]) + + def normalize(payload: dict, base_url: str) -> list[dict]: included = {(item["type"], item["id"]): item for item in payload.get("included") or []} events = [] @@ -79,29 +215,55 @@ def normalize(payload: dict, base_url: str) -> list[dict]: def upload_artifacts( event: dict, gateway_url: str, base_url: str, public_url: str, token: str, correlation: str ) -> list[str]: - ids = [] + ids: list[str] = [] + warnings: list[str] = list(event.get("artifactWarnings") or []) + max_bytes, max_archive_bytes, timeout, retries = _attachment_policy() + temp_root = Path(os.getenv( + "TECHFLOW_COMMUNITY_ATTACHMENT_TMP_DIR", str(Path(tempfile.gettempdir()) / "techflow-community-poller") + )) + temp_root.mkdir(parents=True, exist_ok=True, mode=0o700) for raw_url in event.pop("attachmentUrls", []): public_attachment_url = urllib.parse.urljoin(public_url + "/", raw_url) parsed = urllib.parse.urlparse(public_attachment_url) if parsed.scheme != "https" or parsed.netloc != urllib.parse.urlparse(public_url).netloc: + warnings.append(_warning(Path(parsed.path).name, "origin")) continue internal_url = urllib.parse.urljoin(base_url + "/", parsed.path.lstrip("/")) if parsed.query: internal_url = f"{internal_url}?{parsed.query}" req = urllib.request.Request(internal_url, headers={"Authorization": f"Token {token}"}) - with urllib.request.urlopen(req, timeout=30) as response: - content = response.read(10 * 1024 * 1024 + 1) - media_type = response.headers.get_content_type() - if len(content) > 10 * 1024 * 1024: - continue filename = Path(parsed.path).name or "community-artifact" - upload = urllib.request.Request( - gateway_url.rstrip("/") + "/v1/artifacts", data=content, method="POST", - headers={"Content-Type": media_type, "X-Artifact-Filename": filename, - "X-Artifact-Classification": "D0", "X-Correlation-Id": correlation}, - ) - with urllib.request.urlopen(upload, timeout=30) as response: - ids.append(str(json.loads(response.read().decode("utf-8"))["data"]["artifactId"])) + with tempfile.NamedTemporaryFile(prefix="attachment-", suffix=".part", dir=temp_root, delete=False) as holder: + temporary = Path(holder.name) + temporary.unlink(missing_ok=True) + try: + try: + _, media_type, disposition, filename = _read_attachment( + req, temporary, filename=filename, max_bytes=max_bytes, + max_archive_bytes=max_archive_bytes, timeout=timeout, retries=retries, + ) + filename = _attachment_filename(disposition, parsed.path) + except ValueError as exc: + if str(exc) == "size": + warnings.append(_warning(filename, "size")) + continue + raise + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError): + warnings.append(_warning(filename, "fetch")) + continue + media_type = _normalized_attachment_media_type(filename, media_type) + try: + ids.append(_upload_artifact(gateway_url, temporary, filename, media_type, correlation, timeout)) + except urllib.error.HTTPError as exc: + if exc.code in TRANSIENT_HTTP_STATUSES: + warnings.append(_warning(filename, "fetch")) + else: + warnings.append(_warning(filename, "unsafe")) + except (urllib.error.URLError, TimeoutError): + warnings.append(_warning(filename, "fetch")) + finally: + temporary.unlink(missing_ok=True) + event["artifactWarnings"] = warnings return ids diff --git a/services/ai-gateway/scripts/verify_large_upload_boundaries.py b/services/ai-gateway/scripts/verify_large_upload_boundaries.py new file mode 100644 index 0000000..084b3f7 --- /dev/null +++ b/services/ai-gateway/scripts/verify_large_upload_boundaries.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Generate and upload exact Issue #72 boundary artifacts without loading them into memory.""" + +from __future__ import annotations + +import argparse +import hashlib +import http.client +import json +from pathlib import Path +import time +from urllib.parse import urlsplit +import uuid +import zipfile + + +MIB = 1024 * 1024 +GIB = 1024 * MIB +REGULAR_LIMIT = 1 * GIB +ARCHIVE_LIMIT = 10 * GIB +CHUNK_BYTES = 4 * MIB +LOG_PREFIX = b"2026-08-16T00:00:00Z INFO TechFlow boundary validation completed normally " +LOG_LINE = LOG_PREFIX + (b"x" * (4095 - len(LOG_PREFIX))) + b"\n" + + +def _write_log_bytes(target, size: int, digest=None) -> None: + block = (LOG_LINE * ((CHUNK_BYTES // len(LOG_LINE)) + 1))[:CHUNK_BYTES] + remaining = size + while remaining: + chunk = block[: min(len(block), remaining)] + target.write(chunk) + if digest is not None: + digest.update(chunk) + remaining -= len(chunk) + + +def create_exact_log(path: Path) -> dict[str, object]: + digest = hashlib.sha256() + with path.open("wb") as target: + _write_log_bytes(target, REGULAR_LIMIT, digest) + return {"path": str(path), "sizeBytes": path.stat().st_size, "sha256": digest.hexdigest()} + + +def create_exact_zip(path: Path) -> dict[str, object]: + # Leave room for the central directory and then pad the EOCD comment to the exact 10 GiB boundary. + member_size = ARCHIVE_LIMIT - 65_023 + info = zipfile.ZipInfo("support.log") + info.compress_type = zipfile.ZIP_STORED + info.file_size = member_size + with zipfile.ZipFile(path, "w", allowZip64=True) as archive: + with archive.open(info, "w", force_zip64=True) as target: + _write_log_bytes(target, member_size) + base_size = path.stat().st_size + padding = ARCHIVE_LIMIT - base_size + if not 0 <= padding <= 65_535: + raise RuntimeError(f"ZIP boundary padding is invalid: base={base_size}, padding={padding}") + with zipfile.ZipFile(path, "a", allowZip64=True) as archive: + archive.comment = b"T" * padding + if path.stat().st_size != ARCHIVE_LIMIT: + raise RuntimeError(f"ZIP boundary size mismatch: {path.stat().st_size}") + with zipfile.ZipFile(path, "r", allowZip64=True) as archive: + bad = archive.testzip() + if bad is not None: + raise RuntimeError(f"ZIP integrity failure: {bad}") + extracted = archive.getinfo("support.log").file_size + return {"path": str(path), "sizeBytes": path.stat().st_size, "entryBytes": extracted} + + +def _connection(base_url: str, timeout: int): + parsed = urlsplit(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("base URL must be http(s)") + klass = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return klass(parsed.hostname, port, timeout=timeout), parsed.path.rstrip("/") + + +def upload(base_url: str, path: Path, filename: str, media_type: str, timeout: int) -> dict[str, object]: + connection, prefix = _connection(base_url, timeout) + started = time.monotonic() + connection.putrequest("POST", f"{prefix}/v1/artifacts") + connection.putheader("Content-Type", media_type) + connection.putheader("Content-Length", str(path.stat().st_size)) + connection.putheader("X-Artifact-Filename", filename) + connection.putheader("X-Artifact-Classification", "D0") + connection.putheader("X-Correlation-Id", f"issue72-upload-{uuid.uuid4().hex}") + connection.endheaders() + with path.open("rb") as source: + while chunk := source.read(CHUNK_BYTES): + connection.send(chunk) + response = connection.getresponse() + payload = response.read().decode("utf-8") + elapsed = round(time.monotonic() - started, 3) + if response.status != 201: + raise RuntimeError(f"upload failed: status={response.status}, payload={payload[:500]}") + artifact_id = json.loads(payload)["data"]["artifactId"] + connection.close() + return {"status": response.status, "artifactId": artifact_id, "elapsedSeconds": elapsed} + + +def preflight_rejection(base_url: str, filename: str, media_type: str, size: int, timeout: int) -> dict[str, object]: + connection, prefix = _connection(base_url, timeout) + connection.putrequest("POST", f"{prefix}/v1/artifacts") + connection.putheader("Content-Type", media_type) + connection.putheader("Content-Length", str(size)) + connection.putheader("X-Artifact-Filename", filename) + connection.putheader("X-Artifact-Classification", "D0") + connection.putheader("X-Correlation-Id", f"issue72-preflight-{uuid.uuid4().hex}") + connection.endheaders() + response = connection.getresponse() + payload = response.read().decode("utf-8") + connection.close() + if response.status != 400: + raise RuntimeError(f"oversize preflight was not rejected: status={response.status}, payload={payload[:500]}") + return {"status": response.status, "declaredBytes": size} + + +def delete(base_url: str, artifact_id: str, timeout: int) -> dict[str, object]: + connection, prefix = _connection(base_url, timeout) + headers = { + "Idempotency-Key": f"issue72-delete-{uuid.uuid4().hex}", + "X-Correlation-Id": f"issue72-delete-{uuid.uuid4().hex}", + } + connection.request("DELETE", f"{prefix}/v1/artifacts/{artifact_id}", headers=headers) + response = connection.getresponse() + payload = response.read().decode("utf-8") + connection.close() + if response.status != 200: + raise RuntimeError(f"delete failed: status={response.status}, payload={payload[:500]}") + return {"status": response.status, "artifactId": artifact_id} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--workdir", type=Path, required=True) + parser.add_argument("--timeout", type=int, default=7200) + parser.add_argument("--keep-files", action="store_true") + parser.add_argument("--reuse-files", action="store_true") + args = parser.parse_args() + args.workdir.mkdir(parents=True, exist_ok=True) + log_path = args.workdir / "issue72-exact-1g.log" + zip_path = args.workdir / "issue72-exact-10g.zip" + results: dict[str, object] = {"limits": {"regularBytes": REGULAR_LIMIT, "archiveBytes": ARCHIVE_LIMIT}} + artifact_ids: list[str] = [] + try: + if args.reuse_files: + if log_path.stat().st_size != REGULAR_LIMIT or zip_path.stat().st_size != ARCHIVE_LIMIT: + raise RuntimeError("reused boundary files do not have the exact required sizes") + with zipfile.ZipFile(zip_path, "r", allowZip64=True) as archive: + if archive.testzip() is not None: + raise RuntimeError("reused ZIP boundary file failed integrity validation") + results["generatedRegular"] = {"path": str(log_path), "sizeBytes": log_path.stat().st_size} + results["generatedArchive"] = {"path": str(zip_path), "sizeBytes": zip_path.stat().st_size} + else: + results["generatedRegular"] = create_exact_log(log_path) + results["generatedArchive"] = create_exact_zip(zip_path) + regular = upload(args.base_url, log_path, log_path.name, "text/plain", args.timeout) + artifact_ids.append(str(regular["artifactId"])) + results["regularBoundary"] = regular + archive = upload(args.base_url, zip_path, zip_path.name, "application/zip", args.timeout) + artifact_ids.append(str(archive["artifactId"])) + results["archiveBoundary"] = archive + results["regularOverBoundary"] = preflight_rejection( + args.base_url, "issue72-over-1g.log", "text/plain", REGULAR_LIMIT + 1, args.timeout + ) + results["archiveOverBoundary"] = preflight_rejection( + args.base_url, "issue72-over-10g.zip", "application/zip", ARCHIVE_LIMIT + 1, args.timeout + ) + finally: + results["cleanup"] = [delete(args.base_url, artifact_id, args.timeout) for artifact_id in artifact_ids] + if not args.keep_files: + log_path.unlink(missing_ok=True) + zip_path.unlink(missing_ok=True) + results["result"] = "PASS" + print(json.dumps(results, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai-gateway/tests/test_artifact_maintenance.py b/services/ai-gateway/tests/test_artifact_maintenance.py new file mode 100644 index 0000000..95b913a --- /dev/null +++ b/services/ai-gateway/tests/test_artifact_maintenance.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from uuid import uuid4 + + +SPEC = importlib.util.spec_from_file_location( + "artifact_maintenance", Path(__file__).parents[1] / "scripts" / "artifact_maintenance.py" +) +artifact_maintenance = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(artifact_maintenance) + + +class ArtifactMaintenanceTests(unittest.TestCase): + def test_expired_artifact_is_removed_and_capacity_is_reported(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact_id = uuid4() + (root / f"{artifact_id}.bin").write_bytes(b"expired") + (root / f"{artifact_id}.json").write_text( + json.dumps({ + "artifactId": str(artifact_id), "filename": "expired.log", "mediaType": "text/plain", + "sha256": "ignored", "sizeBytes": 7, "kind": "LOG", "width": None, "height": None, + "entryCount": 1, "extractedBytes": 7, "evidenceTruncated": False, "redactionCount": 0, + "createdAt": (datetime.now(timezone.utc) - timedelta(hours=2)).isoformat(), + "expiresAt": (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat(), + }), encoding="utf-8", + ) + with patch.dict("os.environ", { + "TECHFLOW_ARTIFACT_DISK_WARN_PERCENT": "98", + "TECHFLOW_ARTIFACT_DISK_CRITICAL_PERCENT": "99", + }, clear=False): + result = artifact_maintenance.maintain_once(root) + self.assertEqual(1, result["removed"]) + self.assertIn(result["level"], {"ok", "warning", "critical"}) + self.assertFalse((root / f"{artifact_id}.bin").exists()) + + def test_invalid_capacity_thresholds_are_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory, patch.dict("os.environ", { + "TECHFLOW_ARTIFACT_DISK_WARN_PERCENT": "90", + "TECHFLOW_ARTIFACT_DISK_CRITICAL_PERCENT": "80", + }, clear=False): + with self.assertRaises(RuntimeError): + artifact_maintenance.maintain_once(Path(directory)) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/ai-gateway/tests/test_assist.py b/services/ai-gateway/tests/test_assist.py index a51c7c2..723f6c1 100644 --- a/services/ai-gateway/tests/test_assist.py +++ b/services/ai-gateway/tests/test_assist.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import base64 import gzip from io import BytesIO @@ -196,6 +197,37 @@ def test_octet_stream_log_is_content_validated_and_normalized(self) -> None: self.assertEqual("text/plain", record.media_type) self.assertEqual("LOG", record.kind) + def test_regular_and_archive_upload_boundaries_are_distinct(self) -> None: + archive_buffer = BytesIO() + with zipfile.ZipFile(archive_buffer, "w", zipfile.ZIP_STORED) as archive: + archive.writestr("service.log", "INFO ready\nERROR failed\n" * 8) + self.assertGreater(len(archive_buffer.getvalue()), 64) + with tempfile.TemporaryDirectory() as root: + store = ArtifactStore( + root, retention_hours=1, max_bytes=64, max_archive_bytes=4096, + max_extracted_bytes=8192, max_compression_ratio=100, + ) + with self.assertRaises(InvalidBoundaryError): + store.put("service.log", "text/plain", b"A" * 64 + b"\n") + record = store.put("support.zip", "application/zip", archive_buffer.getvalue()) + self.assertEqual("application/zip", record.media_type) + + def test_stream_content_length_is_rejected_before_body_consumption(self) -> None: + consumed = False + + async def chunks(): + nonlocal consumed + consumed = True + yield b"INFO ready\n" + + with tempfile.TemporaryDirectory() as root: + store = ArtifactStore(root, retention_hours=1, max_bytes=64, max_archive_bytes=256) + with self.assertRaises(InvalidBoundaryError): + asyncio.run(store.put_stream( + "service.log", "text/plain", chunks(), content_length=65, + )) + self.assertFalse(consumed) + class _Responses: def __init__(self) -> None: diff --git a/services/ai-gateway/tests/test_community_poller.py b/services/ai-gateway/tests/test_community_poller.py index 6698dfc..143d6b4 100644 --- a/services/ai-gateway/tests/test_community_poller.py +++ b/services/ai-gateway/tests/test_community_poller.py @@ -1,8 +1,13 @@ from __future__ import annotations import importlib.util +from email.message import Message +from io import BytesIO +import os from pathlib import Path +import tempfile import unittest +from unittest.mock import patch SPEC = importlib.util.spec_from_file_location( @@ -13,6 +18,21 @@ SPEC.loader.exec_module(poll_flarum) +class FakeResponse(BytesIO): + def __init__(self, data: bytes, *, content_type: str = "text/plain", content_length: int | None = None) -> None: + super().__init__(data) + self.headers = Message() + self.headers["Content-Type"] = content_type + if content_length is not None: + self.headers["Content-Length"] = str(content_length) + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + class CommunityPollerTests(unittest.TestCase): def test_only_unanswered_discussions_are_normalized(self) -> None: payload = { @@ -41,6 +61,63 @@ def test_html_parser_does_not_execute_or_expand_markup(self) -> None: parser.feed("

질문

") self.assertEqual(["질문", "ignore()"], parser.text) + def test_attachment_policy_accepts_exact_boundary_and_rejects_one_byte_over(self) -> None: + exact = b"A" * 2048 + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / "a.log" + with patch.object(poll_flarum.urllib.request, "urlopen", return_value=FakeResponse(exact)): + size, media_type, _, _ = poll_flarum._read_attachment( + poll_flarum.urllib.request.Request("https://community.ablecloud.io/a.log"), + destination, filename="a.log", max_bytes=2048, max_archive_bytes=4096, + timeout=5, retries=0, + ) + self.assertEqual(2048, size) + self.assertEqual(exact, destination.read_bytes()) + self.assertEqual("text/plain", media_type) + + with patch.object(poll_flarum.urllib.request, "urlopen", return_value=FakeResponse(exact + b"!")): + with self.assertRaisesRegex(ValueError, "size"): + poll_flarum._read_attachment( + poll_flarum.urllib.request.Request("https://community.ablecloud.io/a.log"), + destination, filename="a.log", max_bytes=2048, max_archive_bytes=4096, + timeout=5, retries=0, + ) + + def test_content_length_is_rejected_before_body_download(self) -> None: + with tempfile.TemporaryDirectory() as directory, patch.object( + poll_flarum.urllib.request, "urlopen", return_value=FakeResponse( + b"", content_type="application/zip", content_length=4097, + ), + ): + with self.assertRaisesRegex(ValueError, "size"): + poll_flarum._read_attachment( + poll_flarum.urllib.request.Request("https://community.ablecloud.io/a.zip"), + Path(directory) / "a.zip", filename="a.zip", max_bytes=2048, + max_archive_bytes=4096, timeout=5, retries=0, + ) + + def test_attachment_policy_environment_is_bounded(self) -> None: + with patch.dict(os.environ, {"TECHFLOW_COMMUNITY_ATTACHMENT_MAX_BYTES": str(1024 * 1024 * 1024 + 1)}, clear=False): + with self.assertRaises(RuntimeError): + poll_flarum._attachment_policy() + + def test_external_attachment_is_skipped_with_understandable_warning(self) -> None: + event = {"attachmentUrls": ["https://example.invalid/secret.log"]} + with tempfile.TemporaryDirectory() as directory, patch.dict( + os.environ, {"TECHFLOW_COMMUNITY_ATTACHMENT_TMP_DIR": directory}, clear=False, + ): + ids = poll_flarum.upload_artifacts( + event, "http://gateway:8090", "http://172.16.0.234", + "https://community.ablecloud.io", "runtime-token", "community-test-0001", + ) + self.assertEqual([], ids) + self.assertIn("Community 외부 주소", event["artifactWarnings"][0]) + + def test_archive_media_type_is_normalized_from_download_filename(self) -> None: + self.assertEqual("application/zip", poll_flarum._normalized_attachment_media_type("support.zip", "application/force-download")) + self.assertEqual("application/gzip", poll_flarum._normalized_attachment_media_type("support.tar.gz", "application/octet-stream")) + self.assertEqual("application/gzip", poll_flarum._normalized_attachment_media_type("agent.log.gz", "application/octet-stream")) + if __name__ == "__main__": unittest.main() diff --git a/services/ai-gateway/tests/test_config.py b/services/ai-gateway/tests/test_config.py index 9559ef2..9f89f88 100644 --- a/services/ai-gateway/tests/test_config.py +++ b/services/ai-gateway/tests/test_config.py @@ -14,6 +14,27 @@ def test_safe_defaults(self) -> None: self.assertEqual("memory", settings.store_backend) self.assertEqual("mock", settings.provider_mode) self.assertEqual(128, settings.embedding_batch_size) + self.assertEqual(1024 * 1024 * 1024, settings.artifact_max_bytes) + self.assertEqual(10 * 1024 * 1024 * 1024, settings.artifact_max_archive_bytes) + self.assertEqual(100 * 1024 * 1024 * 1024, settings.artifact_max_extracted_bytes) + + def test_large_upload_boundary_is_bounded(self) -> None: + Settings( + artifact_max_bytes=1024 * 1024 * 1024, + artifact_max_archive_bytes=10 * 1024 * 1024 * 1024, + artifact_max_extracted_bytes=100 * 1024 * 1024 * 1024, + ).validate() + with self.assertRaises(ConfigurationError): + Settings( + artifact_max_bytes=1024 * 1024 * 1024 + 1, + artifact_max_archive_bytes=10 * 1024 * 1024 * 1024, + artifact_max_extracted_bytes=100 * 1024 * 1024 * 1024, + ).validate() + with self.assertRaises(ConfigurationError): + Settings( + artifact_max_archive_bytes=10 * 1024 * 1024 * 1024 + 1, + artifact_max_extracted_bytes=100 * 1024 * 1024 * 1024, + ).validate() def test_postgres_requires_dsn(self) -> None: with self.assertRaises(ConfigurationError): diff --git a/services/ai-gateway/tests/test_container_contract.py b/services/ai-gateway/tests/test_container_contract.py index 75fc85a..f98dff7 100644 --- a/services/ai-gateway/tests/test_container_contract.py +++ b/services/ai-gateway/tests/test_container_contract.py @@ -9,6 +9,8 @@ REPO = ROOT.parents[1] DOCKERFILE = (ROOT / "Dockerfile").read_text(encoding="utf-8") COMPOSE = (REPO / "deploy" / "compose" / "ai-gateway" / "compose.yml").read_text(encoding="utf-8") +MAIN = (ROOT / "app" / "main.py").read_text(encoding="utf-8") +ARTIFACTS = (ROOT / "app" / "artifacts.py").read_text(encoding="utf-8") class ContainerContractTest(unittest.TestCase): @@ -62,6 +64,16 @@ def test_tree_sitter_parsers_are_prefetched_in_the_image(self) -> None: self.assertIn("scripts/prefetch_parsers.py", DOCKERFILE) self.assertIn("TECHFLOW_TREE_SITTER_CACHE", DOCKERFILE) + def test_large_artifacts_use_streaming_and_separate_archive_boundary(self) -> None: + start = MAIN.index('@application.post("/v1/artifacts"') + end = MAIN.index('@application.get("/v1/artifacts', start) + artifact_route = MAIN[start:end] + self.assertIn("request.stream()", artifact_route) + self.assertNotIn("await request.body()", artifact_route) + self.assertIn("async def put_stream", ARTIFACTS) + self.assertIn("TECHFLOW_ARTIFACT_MAX_ARCHIVE_BYTES", COMPOSE) + self.assertIn("TECHFLOW_COMMUNITY_ARCHIVE_MAX_BYTES", COMPOSE) + if __name__ == "__main__": unittest.main() diff --git a/tools/artifacts/issue-72/README.md b/tools/artifacts/issue-72/README.md new file mode 100644 index 0000000..0c92cc1 --- /dev/null +++ b/tools/artifacts/issue-72/README.md @@ -0,0 +1,18 @@ +# Issue #72 결과물 빌드 + +Issue #72 Community 대용량 첨부 개선 보고서와 발표자료를 재생성한다. + +## 입력 + +- `docs/evidence/issue-72/large-upload-production-validation.json` +- `docs/reports/issue-72-community-large-upload-validation.md` +- `docs/runbooks/community-large-uploads.md` + +## 출력 + +- `output/pdf/techflow-issue-72-large-upload-report.pdf` +- `output/presentation/techflow-issue-72-large-upload.pptx` +- `output/pdf/techflow-issue-72-large-upload-presentation.pdf` +- `output/issue-72-large-upload-artifact-manifest.json` + +실행 환경은 Codex bundled Python/Node와 Presentation artifact-tool을 사용한다. 빌드 후 `validate_artifacts.py`로 페이지 수, 파일 크기, 구조화 증적, 비밀정보 미포함을 검증한다. diff --git a/tools/artifacts/issue-72/build_manifest.py b/tools/artifacts/issue-72/build_manifest.py new file mode 100644 index 0000000..57a1126 --- /dev/null +++ b/tools/artifacts/issue-72/build_manifest.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +from datetime import datetime, timezone +import hashlib, json +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[3] +OUTPUT=ROOT/"output/issue-72-large-upload-artifact-manifest.json" +ARTIFACTS=[ + "deploy/flarum/issue72-large-upload-policy.sh","docs/evidence/issue-72/large-upload-production-validation.json", + "docs/runbooks/community-large-uploads.md","docs/reports/issue-72-community-large-upload-validation.md", + "output/pdf/techflow-issue-72-large-upload-report.pdf","output/presentation/techflow-issue-72-large-upload.pptx", + "output/pdf/techflow-issue-72-large-upload-presentation.pdf","tools/artifacts/issue-72/README.md", + "tools/artifacts/issue-72/build_report.py","tools/artifacts/issue-72/build_presentation.mjs", + "tools/artifacts/issue-72/build_presentation_pdf.py","tools/artifacts/issue-72/build_manifest.py","tools/artifacts/issue-72/validate_artifacts.py"] +items=[] +for relative in ARTIFACTS: + path=ROOT/relative + if not path.is_file(): raise SystemExit(f"missing artifact: {relative}") + body=path.read_bytes(); items.append({"path":relative,"bytes":len(body),"sha256":hashlib.sha256(body).hexdigest()}) +OUTPUT.write_text(json.dumps({"schemaVersion":"1.0","issue":72,"generatedAt":datetime.now(timezone.utc).isoformat(),"artifactCount":len(items),"artifacts":items},ensure_ascii=False,indent=2)+"\n",encoding="utf-8") +print(OUTPUT) diff --git a/tools/artifacts/issue-72/build_presentation.mjs b/tools/artifacts/issue-72/build_presentation.mjs new file mode 100644 index 0000000..7bdfa45 --- /dev/null +++ b/tools/artifacts/issue-72/build_presentation.mjs @@ -0,0 +1,85 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { Presentation, PresentationFile } from "@oai/artifact-tool"; + +const ROOT = process.env.TECHFLOW_ROOT; +if (!ROOT) throw new Error("TECHFLOW_ROOT is required"); +const renderDir = path.join(ROOT, "tmp", "issue72-presentation", "renders"); +const output = path.join(ROOT, "output", "presentation", "techflow-issue-72-large-upload.pptx"); +await fs.mkdir(renderDir, { recursive: true }); +await fs.mkdir(path.dirname(output), { recursive: true }); + +const deck = Presentation.create({ slideSize: { width: 1280, height: 720 } }); +const C = { ink: "#101010", gray: "#5B616B", panel: "#EDEDED", rule: "#B8BCC4", pale: "#EAF5FB", blue: "#3D8DFF", green: "#117A4B", white: "#FFFFFF" }; +const FONT = "Malgun Gothic"; + +function box(slide, name, left, top, width, height, fill=C.white, line=C.rule) { + return slide.shapes.add({ geometry: "rect", name, position: { left, top, width, height }, fill, line: { style: "solid", fill: line, width: 1 } }); +} +function text(slide, name, value, left, top, width, height, size=24, bold=false, color=C.ink, align="left") { + const item=slide.shapes.add({ geometry:"textbox", name, position:{left,top,width,height}, fill:"none", line:{style:"solid",fill:"none",width:0} }); + item.text=value; item.text.style={fontSize:size,bold,color,typeface:FONT,alignment:align}; return item; +} +function title(slide, value, number) { + text(slide, `title-${number}`, value, 42, 34, 1120, 88, 46, true); + text(slide, `page-${number}`, String(number).padStart(2,"0"), 1180, 656, 58, 24, 16, false, C.gray, "right"); +} +function notes(slide) { + slide.speakerNotes.textFrame.setText("[Sources]\n- docs/evidence/issue-72/large-upload-production-validation.json\n- docs/reports/issue-72-community-large-upload-validation.md\n- docs/runbooks/community-large-uploads.md"); +} + +{ + const s=deck.slides.add(); s.background.fill=C.white; + text(s,"cover-kicker","ABLESTACK TECHFLOW · ISSUE #72",42,42,560,42,20,true,C.gray); + text(s,"cover-title","Community 대용량\n첨부 개선 완료",42,190,560,190,66,true); + text(s,"cover-subtitle","실파일 경계 · 디스크 스트리밍 · 운영 검증",42,430,560,70,28,false,C.gray); + box(s,"cover-hero",660,42,578,588,C.pale,C.rule); + text(s,"cover-stat","1 / 10\nGiB",708,160,480,240,84,true,C.blue,"center"); + text(s,"cover-detail","일반 / 압축 파일 상한\n2026.08.16",708,430,480,100,26,false,C.gray,"center"); + notes(s); +} +{ + const s=deck.slides.add(); title(s,"1 GiB / 10 GiB 정책이 전 계층을 관통합니다",2); + const labels=["Nginx\n11 GiB","PHP-FPM\n10 / 11 GiB","Flarum\n1 / 10 GiB","Poller\n디스크 저장","Gateway\n스트리밍","압축 해제\n100 GiB"]; + labels.forEach((label,i)=>{ const left=42+i*199; box(s,`flow-${i}`,left,260,164,150,i===2||i===4?C.pale:C.panel,i===2||i===4?C.blue:C.rule); text(s,`flow-text-${i}`,label,left+12,300,140,70,24,true,C.ink,"center"); if(i<5) text(s,`arrow-${i}`,"→",left+166,313,34,40,30,true,C.gray,"center"); }); + text(s,"flow-foot","7,200초 · 2회 재시도 · 100개 항목 · 압축비 20배 · AI에는 정규화 근거만 전달",42,500,1196,62,24,false,C.gray,"center"); notes(s); +} +{ + const s=deck.slides.add(); title(s,"수신·분석 경계를 함께 확장했습니다",3); + const values=[["계층","적용 전","적용 후"],["Nginx","120 MiB","11 GiB · 7,200초"],["PHP-FPM","120 / 120 MiB","파일 10 / 요청 11 GiB"],["Flarum","50 MiB","일반 1 / 압축 10 GiB"],["Poller","50 MiB · 120초","1 / 10 GiB · 7,200초"],["Gateway","원본 50 / 해제 100 MiB","원본 1/10 · 해제 100 GiB"]]; + const t=s.tables.add({rows:6,columns:3,left:42,top:170,width:1196,height:430,columnWidths:[250,430,516],values}); + t.borders.assign({style:"solid",fill:C.rule,width:1}); + for(let c=0;c<3;c++){t.getCell(0,c).fill="#243B64"; t.getCell(0,c).text.style={fontSize:22,bold:true,color:C.white,typeface:FONT};} + for(let r=1;r<6;r++) for(let c=0;c<3;c++){t.getCell(r,c).fill=r%2?C.white:"#F7F9FC";t.getCell(r,c).text.style={fontSize:19,bold:c===0,color:C.ink,typeface:FONT};} + notes(s); +} +{ + const s=deck.slides.add(); title(s,"정확한 경계 크기로 운영 경로를 검증했습니다",4); + const cards=[{v:"263/263",d:"전체 런타임 회귀"},{v:"1 / 10 GiB",d:"허용 · +1 byte 거부"},{v:"60.3 MiB",d:"10 GiB 분석 최대 메모리"}]; + cards.forEach((card,i)=>{const left=42+i*411;box(s,`metric-${i}`,left,300,374,270,C.panel,C.panel);text(s,`metric-value-${i}`,card.v,left+28,350,318,100,56,true,i===1?C.blue:C.ink,"center");text(s,`metric-detail-${i}`,card.d,left+28,478,318,46,23,false,C.gray,"center");}); + text(s,"metric-caption","Flarum 1 GiB 16초 · 10 GiB 410초 | Gateway 1 GiB 27.751초 · 10 GiB 294.814초",42,160,1196,72,25,false,C.gray,"center"); notes(s); +} +{ + const s=deck.slides.add(); title(s,"위험한 압축과 위장 파일은 모두 닫힌 상태로 거부됩니다",5); + const values=[["시험","HTTP","판정"],["경로 이탈 ZIP","400","거부"],["중첩 압축","400","거부"],["압축 폭탄","400","거부"],["실행 파일 포함","400","거부"],["PNG MIME 위장","400","거부"]]; + const t=s.tables.add({rows:6,columns:3,left:42,top:176,width:760,height:420,columnWidths:[450,130,180],values}); t.borders.assign({style:"solid",fill:C.rule,width:1}); + for(let c=0;c<3;c++){t.getCell(0,c).fill="#243B64";t.getCell(0,c).text.style={fontSize:22,bold:true,color:C.white,typeface:FONT};} + for(let r=1;r<6;r++)for(let c=0;c<3;c++){t.getCell(r,c).fill=r%2?C.white:"#F7F9FC";t.getCell(r,c).text.style={fontSize:20,bold:c===2,color:c===2?C.green:C.ink,typeface:FONT};} + box(s,"security-side",850,176,388,420,C.pale,C.blue); text(s,"security-side-title","검증 후 정리",884,220,320,45,28,true,C.blue); + text(s,"security-side-body","Flarum 첨부 2건 삭제\nGateway Artifact 2건 삭제\n시험 컨테이너 6개 삭제\n시험 볼륨 7개 삭제\n\nDB·파일 잔존\n0건",884,292,320,250,24,false,C.ink); notes(s); +} +{ + const s=deck.slides.add(); s.background.fill=C.white; + text(s,"close-kicker","ISSUE #72 · COMPLETE",42,42,420,42,20,true,C.gray); + text(s,"close-title","운영 상태는\nGO입니다",42,188,850,190,82,true); + text(s,"close-detail","Gateway healthy · Poller failed=0 · Maintainer level=ok\nGitHub→Chat 보호 서비스 frozen / guard passed",42,485,800,85,28,false,C.gray); + box(s,"close-badge",934,208,304,304,C.pale,C.blue); text(s,"close-go","GO",964,296,244,100,72,true,C.blue,"center"); + notes(s); +} + +for (const [index,slide] of deck.slides.items.entries()) { + const blob=await deck.export({slide,format:"png",scale:1}); + await fs.writeFile(path.join(renderDir,`slide-${String(index+1).padStart(2,"0")}.png`),new Uint8Array(await blob.arrayBuffer())); + const layout=await slide.export({format:"layout"}); await fs.writeFile(path.join(renderDir,`slide-${String(index+1).padStart(2,"0")}.layout.json`),await layout.text()); +} +const pptx=await PresentationFile.exportPptx(deck); await pptx.save(output); console.log(output); diff --git a/tools/artifacts/issue-72/build_presentation_pdf.py b/tools/artifacts/issue-72/build_presentation_pdf.py new file mode 100644 index 0000000..d1eecb3 --- /dev/null +++ b/tools/artifacts/issue-72/build_presentation_pdf.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +from pathlib import Path +from PIL import Image +from reportlab.lib.utils import ImageReader +from reportlab.pdfgen import canvas + +ROOT=Path(__file__).resolve().parents[3] +SLIDES=ROOT/"tmp/issue72-presentation/renders" +OUTPUT=ROOT/"output/pdf/techflow-issue-72-large-upload-presentation.pdf" +images=sorted(SLIDES.glob("slide-*.png")) +if len(images)!=6: raise RuntimeError(f"expected 6 slides, found {len(images)}") +with Image.open(images[0]) as first: width,height=first.size +OUTPUT.parent.mkdir(parents=True,exist_ok=True) +pdf=canvas.Canvas(str(OUTPUT),pagesize=(width,height)); pdf.setTitle("TechFlow Issue #72 발표자료"); pdf.setAuthor("ABLESTACK TechFlow") +for image in images: + with Image.open(image) as current: + if current.size!=(width,height): raise ValueError(f"inconsistent slide size: {image}") + pdf.drawImage(ImageReader(str(image)),0,0,width=width,height=height); pdf.showPage() +pdf.save(); print(OUTPUT) diff --git a/tools/artifacts/issue-72/build_report.py b/tools/artifacts/issue-72/build_report.py new file mode 100644 index 0000000..5b15590 --- /dev/null +++ b/tools/artifacts/issue-72/build_report.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Build the Issue #72 production validation report PDF.""" + +from __future__ import annotations + +import json +from pathlib import Path +from xml.sax.saxutils import escape + +from reportlab.lib import colors +from reportlab.lib.pagesizes import A4 +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +from reportlab.lib.units import mm +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont +from reportlab.platypus import BaseDocTemplate, Frame, PageBreak, PageTemplate, Paragraph, Spacer, Table, TableStyle + + +ROOT = Path(__file__).resolve().parents[3] +SOURCE = ROOT / "docs/evidence/issue-72/large-upload-production-validation.json" +OUTPUT = ROOT / "output/pdf/techflow-issue-72-large-upload-report.pdf" +FONT, BOLD = "MalgunGothic", "MalgunGothicBold" +pdfmetrics.registerFont(TTFont(FONT, "C:/Windows/Fonts/malgun.ttf")) +pdfmetrics.registerFont(TTFont(BOLD, "C:/Windows/Fonts/malgunbd.ttf")) + +INK, GRAY = colors.HexColor("#101010"), colors.HexColor("#5B616B") +LINE, BLUE, PALE, GREEN = colors.HexColor("#D4D8DF"), colors.HexColor("#3D8DFF"), colors.HexColor("#EAF5FB"), colors.HexColor("#117A4B") +base = getSampleStyleSheet() +styles = { + "meta": ParagraphStyle("meta", parent=base["Normal"], fontName=BOLD, fontSize=8.5, leading=12, textColor=GRAY), + "title": ParagraphStyle("title", parent=base["Title"], fontName=BOLD, fontSize=24, leading=33, textColor=INK), + "subtitle": ParagraphStyle("subtitle", parent=base["Normal"], fontName=FONT, fontSize=11, leading=18, textColor=GRAY), + "h1": ParagraphStyle("h1", parent=base["Heading1"], fontName=BOLD, fontSize=16, leading=23, textColor=INK, spaceAfter=4*mm), + "h2": ParagraphStyle("h2", parent=base["Heading2"], fontName=BOLD, fontSize=12, leading=18, textColor=INK, spaceBefore=2*mm, spaceAfter=2*mm), + "body": ParagraphStyle("body", parent=base["BodyText"], fontName=FONT, fontSize=9, leading=14.5, textColor=colors.HexColor("#30343B"), spaceAfter=2.2*mm), + "small": ParagraphStyle("small", parent=base["BodyText"], fontName=FONT, fontSize=7.5, leading=11, textColor=GRAY), + "table": ParagraphStyle("table", parent=base["BodyText"], fontName=FONT, fontSize=7.5, leading=10.5, textColor=colors.HexColor("#30343B")), + "table_head": ParagraphStyle("table_head", parent=base["BodyText"], fontName=BOLD, fontSize=7.5, leading=10.5, textColor=colors.white), +} + + +def para(value: object, style: str = "body") -> Paragraph: + return Paragraph(escape(str(value)).replace("\n", "
"), styles[style]) + + +def make_table(rows: list[list[object]], widths: list[float]) -> Table: + cells = [[para(cell, "table_head" if row_index == 0 else "table") for cell in row] for row_index, row in enumerate(rows)] + item = Table(cells, colWidths=widths, repeatRows=1, hAlign="LEFT") + item.setStyle(TableStyle([ + ("BACKGROUND", (0,0), (-1,0), colors.HexColor("#243B64")), ("GRID", (0,0), (-1,-1), .35, LINE), + ("VALIGN", (0,0), (-1,-1), "TOP"), ("LEFTPADDING", (0,0), (-1,-1), 5), + ("RIGHTPADDING", (0,0), (-1,-1), 5), ("TOPPADDING", (0,0), (-1,-1), 5), + ("BOTTOMPADDING", (0,0), (-1,-1), 5), ("ROWBACKGROUNDS", (0,1), (-1,-1), [colors.white, colors.HexColor("#F7F9FC")]), + ])) + return item + + +def callout(text: str, color=GREEN) -> Table: + item = Table([[para(text)]], colWidths=[174*mm]) + item.setStyle(TableStyle([("BACKGROUND",(0,0),(-1,-1),PALE),("BOX",(0,0),(-1,-1),1.25,color), + ("LEFTPADDING",(0,0),(-1,-1),10),("RIGHTPADDING",(0,0),(-1,-1),10), + ("TOPPADDING",(0,0),(-1,-1),8),("BOTTOMPADDING",(0,0),(-1,-1),8)])) + return item + + +def footer(canvas, doc) -> None: + canvas.saveState(); canvas.setFont(FONT, 7); canvas.setFillColor(GRAY) + canvas.drawString(18*mm, 10*mm, "ABLESTACK TechFlow - Issue #72") + canvas.drawRightString(192*mm, 10*mm, f"{doc.page:02d}"); canvas.restoreState() + + +data = json.loads(SOURCE.read_text(encoding="utf-8")); tests=data["tests"]; policy=data["policy"] +OUTPUT.parent.mkdir(parents=True, exist_ok=True) +doc=BaseDocTemplate(str(OUTPUT),pagesize=A4,rightMargin=18*mm,leftMargin=18*mm,topMargin=16*mm,bottomMargin=17*mm, + title="TechFlow Issue #72 Community 대용량 첨부 개선 완료 보고서",author="ABLESTACK TechFlow") +doc.addPageTemplates([PageTemplate(id="normal",frames=[Frame(doc.leftMargin,doc.bottomMargin,doc.width,doc.height,id="content")],onPage=footer)]) + +story=[ + Spacer(1,19*mm), para("ABLESTACK TECHFLOW · ISSUE #72","meta"), Spacer(1,8*mm), + para("Community 대용량 로그·압축파일\n업로드 개선 완료 보고서","title"), Spacer(1,6*mm), + para("일반 파일 1 GiB, 지원 압축 파일 10 GiB를 실제 운영 경로에서 수용하고 초과 파일은 거부하도록 전 계층을 정렬한 결과입니다.","subtitle"), + Spacer(1,17*mm), callout("GO - 운영 적용 완료 · 실파일 1 GiB/10 GiB 경계 통과 · 전체 회귀 263/263"), + Spacer(1,8*mm), para("운영 검증일 2026-08-16 · Flarum 1.8.18 · FoF Upload 1.8.5","meta"), PageBreak(), + para("1. 판단 요약","h1"), + make_table([["항목","결과","판정"],["일반 파일","1 GiB 허용 / +1 byte 거부","PASS"],["압축 파일","10 GiB 허용 / +1 byte 거부","PASS"], + ["압축 분석","100 GiB, 100개, 20배","PASS"],["운영 회귀","263/263","PASS"], + ["정리","첨부·Artifact·시험 자원 잔존 0","PASS"],["보호 서비스","github-chat-v1 frozen","PASS"]],[54*mm,83*mm,37*mm]), + Spacer(1,7*mm), callout("정확한 경계 크기 파일을 직접 전송했고 성공 데이터는 검증 직후 삭제했습니다.",BLUE), PageBreak(), + para("2. 계층별 현재값과 목표값","h1"), + make_table([["계층","적용 전","적용 후"],["Nginx","120 MiB","11 GiB · 7,200초"],["PHP-FPM","120/120 MiB","파일 10 / 요청 11 GiB"], + ["FoF Upload","50 MiB","전역 10 GiB"],["유형 정책","50 MiB","일반 1 / 압축 10 GiB"], + ["Poller","50 MiB · 120초","1/10 GiB · 7,200초 · 2회"],["Gateway","50/100 MiB","원본 1/10 · 해제 100 GiB"]],[43*mm,58*mm,73*mm]), + Spacer(1,6*mm), para("판정 경계는 1,073,741,824바이트와 10,737,418,240바이트로 고정했습니다."), PageBreak(), + para("3. 안전한 처리 흐름","h1"), + make_table([["단계","처리","안전 장치"],["1. Community","10 GiB 요청 수신","일반 1 / 압축 10 GiB"],["2. Poller","1 MiB 단위 디스크 임시 저장","동일 출처 · 2회 재시도"], + ["3. Gateway",".part 스트리밍 + SHA-256","상한 초과 즉시 중단"],["4. 압축 검사","순차 해제 · 근거 선택","100 GiB · 100개 · 20배"], + ["5. AI 질의","정규화 근거만 전달","원본 재파싱 금지"],["6. 보관","24시간 · 15분 정리","70%/85% 용량 이벤트"]],[37*mm,78*mm,59*mm]), + Spacer(1,6*mm), para("10 GiB 압축 분석 중 Gateway 최대 상주 메모리는 약 60.3 MiB로 측정됐습니다."), PageBreak(), + para("4. 실파일 경계 시험","h1"), + make_table([["시험","Flarum","Gateway"],["일반 1 GiB","200 · 16초","201 · 27.751초"],["일반 1 GiB + 1","422 · 저장 0","400 · 선차단"], + ["ZIP 10 GiB","200 · 410초","201 · 294.814초"],["ZIP 10 GiB + 1","413 · 저장 0","400 · 선차단"]],[73*mm,50*mm,51*mm]), + Spacer(1,6*mm), callout("성공 첨부와 Artifact는 모두 삭제했고 Flarum DB와 파일시스템 잔존은 0건입니다.",BLUE), PageBreak(), + para("5. 보안 및 회귀 시험","h1"), + make_table([["시험","Gateway","결과"], + ["경로 이탈 ZIP","400","거부"],["중첩 압축","400","거부"],["압축 폭탄","400","거부"], + ["실행 파일 포함","400","거부"],["PNG MIME 위장","400","거부"]],[66*mm,54*mm,54*mm]), + Spacer(1,6*mm), callout("PR #65 기반 런타임 오버레이 전체 회귀 263/263 통과",GREEN), PageBreak(), + para("6. 운영 상태와 롤백","h1"), + make_table([["항목","상태"],["Gateway","issue-72-large-uploads-1g10g · healthy"],["Poller","반복 처리 failed=0"],["Maintainer","level=ok · 디스크 5%"], + ["Flarum 여유","955 GiB"],["TechFlow 여유","983,218,327,552 bytes"],["보호 서비스","frozen · guard passed"]],[62*mm,112*mm]), + Spacer(1,6*mm), para("Flarum 백업: /var/backups/techflow-flarum/issue72-20260816T010617Z","small"), + para("TechFlow 백업: /home/ablecloud/techflow-ai-gateway/backups/issue72-1g10g-predeploy-20260816T010000Z","small"), + Spacer(1,7*mm), callout("Gateway, Poller, Maintainer만 교체 · DB 스키마 변경 없음",GREEN), PageBreak(), + para("7. 최종 판정","h1"), + callout("Issue #72 완료 조건을 모두 충족했습니다. 운영 상태는 GO입니다.",GREEN), Spacer(1,7*mm), + para("운영자는 일반 파일 1 GiB, 압축 파일 10 GiB, Maintainer level=ok, github-chat-v1 guard passed를 핵심 상태로 확인합니다."), + para("근거 자산","h2"), para("Runbook: docs/runbooks/community-large-uploads.md\n완료 보고서: docs/reports/issue-72-community-large-upload-validation.md\n구조화 증적: docs/evidence/issue-72/large-upload-production-validation.json","small") +] +doc.build(story); print(OUTPUT) diff --git a/tools/artifacts/issue-72/validate_artifacts.py b/tools/artifacts/issue-72/validate_artifacts.py new file mode 100644 index 0000000..49daf91 --- /dev/null +++ b/tools/artifacts/issue-72/validate_artifacts.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import json +from pathlib import Path +from pypdf import PdfReader + +ROOT=Path(__file__).resolve().parents[3] +EVIDENCE=ROOT/"docs/evidence/issue-72/large-upload-production-validation.json" +REPORT=ROOT/"output/pdf/techflow-issue-72-large-upload-report.pdf" +PRESENTATION=ROOT/"output/pdf/techflow-issue-72-large-upload-presentation.pdf" +PPTX=ROOT/"output/presentation/techflow-issue-72-large-upload.pptx" +MANIFEST=ROOT/"output/issue-72-large-upload-artifact-manifest.json" +data=json.loads(EVIDENCE.read_text(encoding="utf-8")); tests=data["tests"] +assert data["issue"]==72 +assert data["policy"]["regularMaxBytes"]==1073741824 +assert data["policy"]["archiveMaxBytes"]==10737418240 +assert data["policy"]["extractedMaxBytes"]==107374182400 +assert tests["runtimeRegression"]=={"total":263,"passed":263} +assert [item["status"] for item in tests["flarumBoundary"]]==[200,422,200,413] +assert [item["status"] for item in tests["gatewayBoundary"]]==[201,400,201,400] +assert tests["cleanup"]["flarumUploadRows"]==0 and tests["cleanup"]["temporaryFilesDeleted"] is True +assert all(item["result"]=="PASS" for item in tests["security"]) +assert tests["protectedService"]["guard"]=="passed" +assert len(PdfReader(str(REPORT)).pages)>=7 and len(PdfReader(str(PRESENTATION)).pages)==6 +assert PPTX.stat().st_size>25_000 +manifest=json.loads(MANIFEST.read_text(encoding="utf-8")); assert manifest["issue"]==72 and manifest["artifactCount"]==len(manifest["artifacts"]) +texts="\n".join((ROOT/p).read_text(encoding="utf-8") for p in ["docs/reports/issue-72-community-large-upload-validation.md","docs/runbooks/community-large-uploads.md"]) +assert not any(token in texts for token in ["Ablecloud1!","Pdh1974","sk-proj-"]) +print("Issue #72 artifacts: PASS")