fix(observability): harden audit and worker telemetry

package.json

@@ -122,7 +122,7 @@
     "typescript": "^6.0.3",
     "vite": "8.2.1",
     "vitest": "^4.1.10",
-    "wrangler": "4.123.0",
+    "wrangler": "4.113.0",
     "zod-openapi": "^6.0.1"
   },
   "patchedDependencies": {

src/features/admin/server/admin-page-auth.ts

@@ -1,4 +1,5 @@
 import { error, redirect } from "@sveltejs/kit";
+import { logAdminSecurityEvent } from "@/lib/audit/security-events";
 import {
   buildReauthenticationPageUrl,
   buildSignInPageUrl,
@@ -23,6 +24,7 @@ export async function requireAdminPage(
   const { getSessionFromHeaders } = await import("@/lib/auth/core");
   const session = await getSessionFromHeaders(request.headers);
   if (!session?.user?.id) {
+    logAdminSecurityEvent(request, "unauthenticated");
     const url = new URL(request.url);
     throw redirect(303, buildSignInPageUrl(`${url.pathname}${url.search}`));
   }
@@ -33,16 +35,23 @@ export async function requireAdminPage(
     select: { id: true, isAdmin: true, name: true, username: true },
   });
 
-  if (!user?.isAdmin) error(404, "Not found");
+  if (!user?.isAdmin) {
+    logAdminSecurityEvent(request, "not_admin");
+    error(404, "Not found");
+  }
   if (options.requireActive) {
     const suspension = await findActiveSuspension(user.id);
-    if (suspension) error(403, "Suspended");
+    if (suspension) {
+      logAdminSecurityEvent(request, "suspended");
+      error(403, "Suspended");
+    }
   }
   if (options.requireRecent) {
     const recent = await resolveAuthoritativeRecentSession(request.headers, {
       expectedUserId: user.id,
     });
     if (!recent.ok) {
+      logAdminSecurityEvent(request, "recent_auth_required");
       const url = new URL(request.url);
       throw redirect(
         303,

src/features/calendar/server/calendar-export-cache.ts

@@ -137,14 +137,16 @@ async function persistStoredCalendar(
   entry: StoredUserCalendarExport,
 ) {
   const namespace = getCloudflareCalendarExportsNamespace();
-  if (!namespace) return;
+  if (!namespace) return true;
 
   try {
     await namespace.put(cacheKey(userId), JSON.stringify(entry), {
       expirationTtl: USER_CALENDAR_EXPORT_KV_EXPIRATION_TTL_SECONDS,
     });
+    return true;
   } catch {
     recordCalendarFeedCacheStatus("store_error");
+    return false;
   }
 }
 
@@ -166,11 +168,19 @@ export async function storeBuiltUserCalendarExport(
   pruneOldestEntries();
   const persistence = persistStoredCalendar(userId, stored);
   if (options.defer) {
-    options.defer(persistence);
+    const deferredPersistence = persistence.then((persisted) => {
+      if (!persisted) {
+        throw new Error("Calendar export cache persistence failed");
+      }
+      recordCalendarFeedCacheStatus("refresh_success");
+    });
+    options.defer(deferredPersistence);
   } else {
-    await persistence;
+    if (!(await persistence)) {
+      throw new Error("Calendar export cache persistence failed");
+    }
+    recordCalendarFeedCacheStatus("refresh_success");
   }
-  recordCalendarFeedCacheStatus("refresh_success");
   return stored;
 }
 
@@ -183,7 +193,13 @@ function refreshUserCalendarExport(
   if (pending) return pending;
 
   const refresh = (async () => {
-    const calendar = await buildExport();
+    let calendar: UserCalendarExport | null;
+    try {
+      calendar = await buildExport();
+    } catch (error) {
+      recordCalendarFeedCacheStatus("refresh_error");
+      throw error;
+    }
     if (!calendar) return null;
     return storeBuiltUserCalendarExport(userId, calendar, { defer });
   })();
@@ -196,9 +212,23 @@ function refreshUserCalendarExport(
   return refresh;
 }
 
-function scheduleStaleCalendarExportRebuild(userId: string) {
-  void enqueueUserCalendarExportRebuild(userId).catch(() => {
-    // Stale-serve path must never fail because enqueue failed.
+function scheduleStaleCalendarExportRebuild(
+  userId: string,
+  defer?: (promise: Promise<unknown>) => void,
+) {
+  const enqueue = enqueueUserCalendarExportRebuild(userId);
+  if (defer) {
+    try {
+      defer(enqueue);
+      return;
+    } catch {
+      // A failed scheduler must not turn a stale response into an error.
+    }
+  }
+
+  enqueue.catch(() => {
+    // The enqueue helper records a low-cardinality failure metric. Keep this
+    // no-defer path non-blocking without leaving an unhandled rejection.
   });
 }
 
@@ -224,7 +254,7 @@ export async function getCachedUserCalendarExport(
     if (ageMs <= USER_CALENDAR_EXPORT_STALE_TTL_MS) {
       // Serve stale immediately and enqueue a Queue rebuild. Do not rebuild ICS
       // on the request path (or inside waitUntil) — that path hit cpu_ms / cancel.
-      scheduleStaleCalendarExportRebuild(userId);
+      scheduleStaleCalendarExportRebuild(userId, options.defer);
       recordCalendarFeedCacheStatus("stale");
       return {
         calendar: cached,

src/features/calendar/server/calendar-export-queue.ts

@@ -1,4 +1,7 @@
-import { getCloudflareCalendarExportRebuildQueue } from "@/lib/adapters/cloudflare-runtime";
+import {
+  getCloudflareCalendarExportRebuildQueue,
+  getCloudflareRuntimeTaskScheduler,
+} from "@/lib/adapters/cloudflare-runtime";
 import { writeCalendarExportRebuildAnalytics } from "@/lib/metrics/analytics-engine";
 
 export type CalendarExportRebuildUserMessage = {
@@ -60,39 +63,84 @@ async function deliverCalendarExportRebuildMessage(
   }
 
   const queue = getCloudflareCalendarExportRebuildQueue();
-  if (queue) {
-    await queue.send(message);
-    writeCalendarExportRebuildAnalytics({ status: "enqueued" });
-    return;
+  if (!queue) {
+    throw new Error("CALENDAR_EXPORT_REBUILD binding is required");
   }
 
-  // Node / vitest without a Queue binding: rebuild in-process.
-  const { processCalendarExportRebuildMessage } = await import(
-    "./calendar-export-rebuild"
-  );
-  await processCalendarExportRebuildMessage(message);
+  await queue.send(message);
   writeCalendarExportRebuildAnalytics({ status: "enqueued" });
 }
 
+function recordCalendarExportRebuildEnqueueFailure() {
+  writeCalendarExportRebuildAnalytics({ status: "enqueue_error" });
+}
+
+function observeEnqueueFailure(enqueue: Promise<void>) {
+  enqueue.catch(() => {
+    // The enqueue function records the low-cardinality failure metric before
+    // rethrowing. This rejection handler keeps no-defer callers from creating
+    // an unhandled rejection while preserving immediate stale responses.
+  });
+}
+
 export async function enqueueUserCalendarExportRebuild(userId: string) {
   const trimmed = userId.trim();
   if (!trimmed) return;
-  await deliverCalendarExportRebuildMessage({ type: "user", userId: trimmed });
+  try {
+    await deliverCalendarExportRebuildMessage({
+      type: "user",
+      userId: trimmed,
+    });
+  } catch (error) {
+    recordCalendarExportRebuildEnqueueFailure();
+    throw error;
+  }
 }
 
 export async function enqueueSectionCalendarExportRebuild(sectionId: number) {
   if (!Number.isInteger(sectionId) || sectionId <= 0) return;
-  await deliverCalendarExportRebuildMessage({ type: "section", sectionId });
+  try {
+    await deliverCalendarExportRebuildMessage({ type: "section", sectionId });
+  } catch (error) {
+    recordCalendarExportRebuildEnqueueFailure();
+    throw error;
+  }
 }
 
-export function scheduleUserCalendarExportRebuild(userId: string) {
-  void enqueueUserCalendarExportRebuild(userId).catch(() => {
-    // Enqueue failures must not fail the write path.
-  });
+export function scheduleUserCalendarExportRebuild(
+  userId: string,
+  defer:
+    | ((promise: Promise<unknown>) => void)
+    | undefined = getCloudflareRuntimeTaskScheduler(),
+) {
+  const enqueue = enqueueUserCalendarExportRebuild(userId);
+  if (defer) {
+    try {
+      defer(enqueue);
+      return;
+    } catch {
+      // A failed scheduler cannot retain the promise. Attach a rejection
+      // observer so the write path remains non-blocking and the enqueue
+      // failure remains visible through its metric.
+    }
+  }
+  observeEnqueueFailure(enqueue);
 }
 
-export function scheduleSectionCalendarExportRebuild(sectionId: number) {
-  void enqueueSectionCalendarExportRebuild(sectionId).catch(() => {
-    // Enqueue failures must not fail the write path.
-  });
+export function scheduleSectionCalendarExportRebuild(
+  sectionId: number,
+  defer:
+    | ((promise: Promise<unknown>) => void)
+    | undefined = getCloudflareRuntimeTaskScheduler(),
+) {
+  const enqueue = enqueueSectionCalendarExportRebuild(sectionId);
+  if (defer) {
+    try {
+      defer(enqueue);
+      return;
+    } catch {
+      // See the user-scoped scheduler path above.
+    }
+  }
+  observeEnqueueFailure(enqueue);
 }

src/features/calendar/server/calendar-export-rebuild.ts

@@ -10,9 +10,23 @@ import { logAppEvent } from "@/lib/log/app-logger";
 import { writeCalendarExportRebuildAnalytics } from "@/lib/metrics/analytics-engine";
 
 export async function rebuildUserCalendarExport(userId: string) {
-  const user = await getUserCalendarRecord(userId);
+  let user: Awaited<ReturnType<typeof getUserCalendarRecord>>;
+  try {
+    user = await getUserCalendarRecord(userId);
+  } catch (error) {
+    writeCalendarExportRebuildAnalytics({ status: "refresh_error" });
+    throw error;
+  }
   if (!user) return null;
-  const calendar = await buildUserCalendarExport(user, userId);
+
+  let calendar: Awaited<ReturnType<typeof buildUserCalendarExport>>;
+  try {
+    calendar = await buildUserCalendarExport(user, userId);
+  } catch (error) {
+    writeCalendarExportRebuildAnalytics({ status: "refresh_error" });
+    throw error;
+  }
+
   return storeBuiltUserCalendarExport(userId, calendar);
 }
 
@@ -90,7 +104,15 @@ export async function handleCalendarExportRebuildBatch(
   for (const message of batch.messages) {
     const body = parseCalendarExportRebuildMessage(message.body);
     if (!body) {
-      message.ack();
+      logAppEvent("error", "calendar-export-rebuild.invalid-message", {
+        event: "calendar-export-rebuild.invalid-message",
+        phase: "consumer",
+        reason: "invalid_envelope",
+        source: "calendar-export-rebuild",
+      });
+      // Keep the body out of logs. Retrying lets the configured DLQ retain the
+      // invalid envelope for bounded operational inspection.
+      message.retry();
       continue;
     }
     parsed.push(body);

src/features/oauth/server/oauth-consent-action.ts

@@ -484,10 +484,13 @@ async function createDeniedOAuthAuthorization(input: {
 }
 
 export async function submitOAuthConsentAction({
+  locals,
   request,
 }: {
+  locals?: { requestId?: string };
   request: Request;
 }) {
+  const requestId = locals?.requestId;
   assertTrustedCookieRequestOrigin(request);
 
   const form = await request.formData();
@@ -515,7 +518,10 @@ export async function submitOAuthConsentAction({
     if (!accept) {
       redirectTarget =
         (await createDeniedOAuthAuthorization({
-          audit: { channel: "web", ...getAuditRequestMetadata(request) },
+          audit: {
+            channel: "web",
+            ...getAuditRequestMetadata(request, requestId),
+          },
           authorizeQuery,
           session,
         })) ?? undefined;
@@ -544,7 +550,7 @@ export async function submitOAuthConsentAction({
         targetType: existingConsent ? "oauth_consent" : "oauth_client",
         userId: session.user.id,
         metadata: { reason: "operation_failed" },
-        ...getAuditRequestMetadata(request),
+        ...getAuditRequestMetadata(request, requestId),
       };
       const recent = await resolveAuthoritativeRecentSession(request.headers, {
         expectedUserId: session.user.id,
@@ -563,7 +569,7 @@ export async function submitOAuthConsentAction({
           targetType: existingConsent ? "oauth_consent" : "oauth_client",
           userId: session.user.id,
           metadata: { reason: recent.reason },
-          ...getAuditRequestMetadata(request),
+          ...getAuditRequestMetadata(request, requestId),
         });
         throw new OAuthRecentAuthRequiredError();
       }
@@ -588,7 +594,7 @@ export async function submitOAuthConsentAction({
         acceptedScopes: uniqueScopes(scope),
         audit: {
           channel: "web",
-          ...getAuditRequestMetadata(request),
+          ...getAuditRequestMetadata(request, requestId),
         },
         authorizeQuery,
         session,