From 6b4c2f82d69caf9ad187c3810b0cebd0028e5bf7 Mon Sep 17 00:00:00 2001 From: Mikaal Naik Date: Tue, 8 Sep 2026 18:06:51 -0400 Subject: [PATCH] Instrument the Toronto voter survey for PostHog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completion and stage drop-off were both invisible: the only event the survey sent was `survey_submitted`, which says how many finished and nothing about how many started or where the rest went. Adds useSurveyAnalytics, which holds every event in one place so they carry the same identity — survey, version, election — plus elapsed seconds. A funnel is only as good as that consistency: a step event missing survey_version pools answers to two question sets into one drop-off number. `survey_step_completed` fires on clearing a step rather than on being shown one, so a respondent who lands on a step and leaves emits nothing for it — which is the drop we want counted. Repeat it as successive funnel steps filtered on step_index to read stage drop-off. `survey_step_blocked` separates "couldn't get past validation" from "gave up", and `survey_results_viewed` separates completions shown a real candidate comparison from wards with no published answers. `survey_submitted` moves out of submitSurvey into the hook, keeping its name and properties, so it carries the step and timing properties the completion funnel is built on. posthog.identify stays where the email is known to have been accepted. Co-Authored-By: Claude Opus 5 (1M context) --- .../toronto/vote/2026/survey/SurveyClient.tsx | 77 ++++++- src/app/toronto/vote/2026/survey/analytics.ts | 210 ++++++++++++++++++ .../toronto/vote/2026/survey/submitSurvey.ts | 10 +- 3 files changed, 288 insertions(+), 9 deletions(-) create mode 100644 src/app/toronto/vote/2026/survey/analytics.ts diff --git a/src/app/toronto/vote/2026/survey/SurveyClient.tsx b/src/app/toronto/vote/2026/survey/SurveyClient.tsx index d6e23ed..70419a6 100644 --- a/src/app/toronto/vote/2026/survey/SurveyClient.tsx +++ b/src/app/toronto/vote/2026/survey/SurveyClient.tsx @@ -1,6 +1,12 @@ "use client"; -import { useEffect, useMemo, useState, type CSSProperties } from "react"; +import { + useEffect, + useMemo, + useRef, + useState, + type CSSProperties, +} from "react"; import Link from "next/link"; import { ArrowRight } from "lucide-react"; @@ -19,6 +25,7 @@ import AlignmentResults, { type SurveyRosterCandidate, } from "./AlignmentResults"; import { submitSurvey, type SurveySubmission } from "./submitSurvey"; +import { useSurveyAnalytics, type SurveyProgress } from "./analytics"; export type SurveyAnswers = Record; @@ -133,6 +140,8 @@ export default function SurveyClient({ const [done, setDone] = useState(false); const [submission, setSubmission] = useState(null); + const analytics = useSurveyAnalytics(survey); + // Everything about the shape of the form comes from the fetched survey, so a // question added in the CMS shows up here with no change to this component. const steps = survey.steps; @@ -142,6 +151,30 @@ export default function SurveyClient({ const isLastStep = step === stepCount - 1; const currentStep = steps[step]; + /* Where the respondent is, for whichever event is about to be sent. Built + from the answers as they stand at the call site rather than from state + read a render later, so a step event can't report the count from before + the answer that triggered it. */ + const progressAt = ( + stepIndex: number, + given: SurveyAnswers = answers, + ): SurveyProgress => { + const target = steps[stepIndex]; + const answered = (ids: string[]) => + ids.filter((id) => (given[id] ?? "").trim().length > 0).length; + + return { + stepIndex, + stepId: target.id, + stepTitle: target.title, + answeredOnStep: answered(target.questions.map((q) => q.id)), + questionsOnStep: target.questions.length, + answeredTotal: answered( + steps.flatMap((s) => s.questions.map((q) => q.id)), + ), + }; + }; + /* The comparison against the ward's candidates. Keyed on the ward the API actually recorded rather than one re-derived here, so the results a respondent reads are the results filed under their response. @@ -239,7 +272,30 @@ export default function SurveyClient({ return { races, wardLabel }; }, [ward, responses, roster, mayoral, survey, answers, wardNames]); + /* What the respondent actually ended up seeing. Held until the ward's + answers have resolved one way or the other — reported the moment they are + done, since a comparison that is still loading is not yet an outcome. A + ward that never resolved has nothing to wait for. */ + const reportedResults = useRef(false); + useEffect(() => { + if (!done || reportedResults.current) return; + if (ward && responses === null) return; + reportedResults.current = true; + analytics.resultsViewed({ + hasComparison: Boolean(comparison), + ward: ward ?? null, + races: comparison?.races.map((race) => race.key) ?? [], + }); + }, [done, ward, responses, comparison, analytics]); + const set = (id: string, value: string) => { + // First answer touched is the start of the run; the hook only lets the + // first of these through. The projected answers are for the event only — + // the state update stays functional, so two changes in one tick can't + // drop each other. + if (value.trim().length > 0) { + analytics.started(progressAt(step, { ...answers, [id]: value })); + } setAnswers((prev) => ({ ...prev, [id]: value })); // Clear the error as soon as they start fixing it; it comes back on Next. setErrors((prev) => { @@ -261,11 +317,18 @@ export default function SurveyClient({ if (message) found[question.id] = message; } setErrors(found); - return Object.keys(found).length === 0; + if (Object.keys(found).length > 0) { + // A step people repeatedly fail to clear looks the same in a funnel as a + // step they lose interest in; this is what tells the two apart. + analytics.stepBlocked(progressAt(step), Object.keys(found)); + return false; + } + return true; }; const next = () => { if (!validateStep()) return; + analytics.stepCompleted(progressAt(step)); setStep((s) => Math.min(stepCount - 1, s + 1)); scrollTop(); }; @@ -273,6 +336,7 @@ export default function SurveyClient({ const back = () => { setErrors({}); setSubmitError(null); + analytics.stepBack(progressAt(step)); setStep((s) => Math.max(0, s - 1)); scrollTop(); }; @@ -282,11 +346,18 @@ export default function SurveyClient({ setSubmitting(true); setSubmitError(null); try { - setSubmission(await submitSurvey(survey, answers)); + const recorded = await submitSurvey(survey, answers); + // The last step is cleared by submitting it, so it gets the same step + // event as every other one — without it the funnel's final step is + // missing and the last screen reads as a total drop. + analytics.stepCompleted(progressAt(step)); + analytics.submitted(progressAt(step), recorded); + setSubmission(recorded); setDone(true); scrollTop(); } catch { // Answers stay on screen so they can just press submit again. + analytics.submitFailed(progressAt(step)); setSubmitError("Something went wrong. Please try again."); } finally { setSubmitting(false); diff --git a/src/app/toronto/vote/2026/survey/analytics.ts b/src/app/toronto/vote/2026/survey/analytics.ts new file mode 100644 index 0000000..399bb72 --- /dev/null +++ b/src/app/toronto/vote/2026/survey/analytics.ts @@ -0,0 +1,210 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useRef } from "react"; +import posthog from "posthog-js"; + +import { DEFAULT_ELECTION_SLUG } from "@/lib/elections/registry"; +import type { Survey } from "@/lib/elections/survey"; + +import type { SurveySubmission } from "./submitSurvey"; + +/** + * PostHog instrumentation for the voter survey, kept in one place so every + * event carries the same identifying properties — which survey, which version, + * which election. A funnel built on these is only as good as that consistency: + * a step event missing `survey_version` silently pools answers to two different + * question sets into one drop-off number. + * + * The two questions this is meant to answer: + * + * completion — `survey_viewed` → `survey_started` → `survey_submitted` + * drop-off — `survey_step_completed`, filtered to `step_index` 0, 1, 2 … + * as successive funnel steps + * + * `survey_step_completed` fires once per step actually cleared, so the funnel + * reads as "reached the end of step n" rather than "was shown step n". A + * respondent who lands on step 3 and leaves without answering never emits it, + * which is the drop we want counted. + */ + +/** Where someone is in the questionnaire when an event fires. */ +export type SurveyProgress = { + /** zero-based, so PostHog funnel steps line up with the array */ + stepIndex: number; + stepId: string; + stepTitle: string; + /** questions on this step that have a non-empty answer */ + answeredOnStep: number; + questionsOnStep: number; + /** answered across the whole survey so far */ + answeredTotal: number; +}; + +function progressProps(progress: SurveyProgress) { + return { + step_index: progress.stepIndex, + // Step number as a human reads it — the funnel is filtered on step_index, + // but a breakdown table is unreadable without this. + step_number: progress.stepIndex + 1, + step_id: progress.stepId, + step_title: progress.stepTitle, + answered_on_step: progress.answeredOnStep, + questions_on_step: progress.questionsOnStep, + answered_total: progress.answeredTotal, + }; +} + +export type SurveyAnalytics = { + /** First answer touched. Fires once; later calls are ignored. */ + started: (progress: SurveyProgress) => void; + /** A step validated and the respondent moved on. */ + stepCompleted: (progress: SurveyProgress) => void; + /** Validation held them on the step. `fields` are the question ids at fault. */ + stepBlocked: (progress: SurveyProgress, fields: string[]) => void; + stepBack: (progress: SurveyProgress) => void; + submitted: (progress: SurveyProgress, submission: SurveySubmission) => void; + submitFailed: (progress: SurveyProgress) => void; + /** The comparison view rendered — or didn't, for want of a ward or answers. */ + resultsViewed: (props: { + hasComparison: boolean; + ward: string | null; + races: string[]; + }) => void; +}; + +export function useSurveyAnalytics(survey: Survey): SurveyAnalytics { + const base = useMemo( + () => ({ + survey: survey.slug, + survey_version: survey.version, + election: DEFAULT_ELECTION_SLUG, + step_count: survey.steps.length, + }), + [survey], + ); + + // Wall-clock from the moment the questionnaire was rendered. Reported in + // seconds because nothing here is measured finely enough to justify ms. + // Stamped in the mount effect rather than here: reading the clock during + // render is impure, and the survey isn't on screen until the effect runs. + const openedAt = useRef(0); + const hasStarted = useRef(false); + /** Set once the run is accounted for — submitted, or already reported as + * abandoned — so the unload handler can't file a second ending for it. */ + const settled = useRef(false); + /** Latest progress, read by the unload handler, which has no other way to + * know where they got to. */ + const latest = useRef(null); + + const secondsElapsed = () => + openedAt.current === 0 + ? 0 + : Math.round((Date.now() - openedAt.current) / 1000); + + const capture = useCallback( + (event: string, props: Record = {}) => { + posthog.capture(event, { + ...base, + seconds_elapsed: secondsElapsed(), + ...props, + }); + }, + [base], + ); + + // Top of the funnel. Page views are autocaptured, but a view of *this* + // survey at *this* version is not something a URL alone says, and the + // completion rate is measured against it. + useEffect(() => { + openedAt.current = Date.now(); + posthog.capture("survey_viewed", base); + }, [base]); + + // The drop itself. A respondent who leaves mid-survey never sends anything + // else, so this is the only event that says how far they got before going. + // sendBeacon because the page is on its way out; visibilitychange as well as + // pagehide, since a backgrounded mobile tab may never unload. + // + // That makes it a best-effort last-seen marker rather than a verdict: + // someone who switches tabs and comes back to finish emits this *and* + // `survey_submitted`. Read completion from `survey_submitted` and drop-off + // from the absence of the next `survey_step_completed` — not from a count of + // these. At most one is sent per run, so it can't inflate on tab-switching. + useEffect(() => { + const report = () => { + if (settled.current || !hasStarted.current) return; + settled.current = true; + const progress = latest.current; + posthog.capture( + "survey_abandoned", + { + ...base, + seconds_elapsed: secondsElapsed(), + ...(progress ? progressProps(progress) : {}), + }, + { transport: "sendBeacon" }, + ); + }; + + const onHide = () => { + if (document.visibilityState === "hidden") report(); + }; + + document.addEventListener("visibilitychange", onHide); + window.addEventListener("pagehide", report); + return () => { + document.removeEventListener("visibilitychange", onHide); + window.removeEventListener("pagehide", report); + }; + }, [base]); + + return useMemo( + () => ({ + started(progress) { + latest.current = progress; + if (hasStarted.current) return; + hasStarted.current = true; + capture("survey_started", progressProps(progress)); + }, + stepCompleted(progress) { + latest.current = progress; + capture("survey_step_completed", progressProps(progress)); + }, + stepBlocked(progress, fields) { + latest.current = progress; + capture("survey_step_blocked", { + ...progressProps(progress), + fields, + field_count: fields.length, + }); + }, + stepBack(progress) { + latest.current = progress; + capture("survey_step_back", progressProps(progress)); + }, + submitted(progress, submission) { + latest.current = progress; + settled.current = true; + capture("survey_submitted", { + ...progressProps(progress), + ward: submission.derivedRegion ?? submission.region ?? null, + }); + }, + submitFailed(progress) { + latest.current = progress; + capture("survey_submit_failed", progressProps(progress)); + }, + resultsViewed({ hasComparison, ward, races }) { + // Completing and being shown a comparison are different outcomes: a + // ward whose candidates haven't answered gets a thank-you and nothing + // to read, and that is worth being able to count. + capture("survey_results_viewed", { + has_comparison: hasComparison, + ward, + races, + }); + }, + }), + [capture], + ); +} diff --git a/src/app/toronto/vote/2026/survey/submitSurvey.ts b/src/app/toronto/vote/2026/survey/submitSurvey.ts index 1d15430..c866d0c 100644 --- a/src/app/toronto/vote/2026/survey/submitSurvey.ts +++ b/src/app/toronto/vote/2026/survey/submitSurvey.ts @@ -74,15 +74,13 @@ export async function submitSurvey( const data = await res.json().catch(() => ({})); + // Identify here rather than in the caller: this is where the email is known + // to have been accepted. The `survey_submitted` event itself is captured by + // useSurveyAnalytics, which also holds the step and timing properties the + // completion funnel is built on — one event, one place that emits it. if (answers.email) { posthog.identify(answers.email, { email: answers.email }); } - posthog.capture("survey_submitted", { - survey: survey.slug, - survey_version: survey.version, - election: DEFAULT_ELECTION_SLUG, - ward: data.derivedRegion ?? data.region ?? null, - }); return { surveySlug: data.surveySlug ?? survey.slug,