Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 47 additions & 1 deletion packages/core/src/audioGroups.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import { beforeEach, describe, expect, it } from "vitest";
import { audioGroupOf, HF_AUDIO_GROUP_ATTR, resolveAudioGroups } from "./audioGroups.js";
import {
audioGroupOf,
HF_AUDIO_GROUP_ATTR,
resolveAudioGroups,
resolveCarveSourceIds,
} from "./audioGroups.js";

beforeEach(() => {
document.body.innerHTML = "";
Expand Down Expand Up @@ -64,6 +69,47 @@ describe("audioGroupOf", () => {
});
});

describe("resolveCarveSourceIds", () => {
it("expands a group id to its current members", () => {
document.body.innerHTML = `
<audio id="vo-1" data-audio-group="voiceover"></audio>
<audio id="vo-2" data-audio-group="voiceover"></audio>
`;
expect(resolveCarveSourceIds(document, ["voiceover"])).toEqual(["vo-1", "vo-2"]);
});

it("picks up a member added to the group after the carve was set (analysis-time, not frozen)", () => {
document.body.innerHTML = `
<audio id="vo-1" data-audio-group="voiceover"></audio>
<audio id="vo-2" data-audio-group="voiceover"></audio>
`;
expect(resolveCarveSourceIds(document, ["voiceover"])).toEqual(["vo-1", "vo-2"]);
document.body.insertAdjacentHTML(
"beforeend",
`<audio id="vo-3" data-audio-group="voiceover"></audio>`,
);
expect(resolveCarveSourceIds(document, ["voiceover"])).toEqual(["vo-1", "vo-2", "vo-3"]);
});

it("passes through a plain clip id that still exists", () => {
document.body.innerHTML = `<audio id="vo-1"></audio>`;
expect(resolveCarveSourceIds(document, ["vo-1"])).toEqual(["vo-1"]);
});

it("drops an id that resolves to nothing — a deleted clip, an empty or vanished group", () => {
document.body.innerHTML = `<audio id="vo-1"></audio>`;
expect(resolveCarveSourceIds(document, ["vo-1", "deleted", "no-such-group"])).toEqual(["vo-1"]);
});

it("dedupes and preserves first-seen order across a mix of group and plain ids", () => {
document.body.innerHTML = `
<audio id="vo-1" data-audio-group="voiceover"></audio>
<audio id="vo-2" data-audio-group="voiceover"></audio>
`;
expect(resolveCarveSourceIds(document, ["voiceover", "vo-1"])).toEqual(["vo-1", "vo-2"]);
});
});

describe(HF_AUDIO_GROUP_ATTR, () => {
it("is the attribute name membership is keyed on", () => {
expect(HF_AUDIO_GROUP_ATTR).toBe("data-audio-group");
Expand Down
31 changes: 31 additions & 0 deletions packages/core/src/audioGroups.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,37 @@ export function resolveAudioGroups(root: ParentNode): HfAudioGroup[] {
return groups;
}

/**
* Expand a list of source ids for a carve: a plain id passes through if it
* still exists, a group id expands to its CURRENT members. Resolved fresh
* every time — group membership is never frozen into the carve's own
* attribute, so adding a fourth voice to a group already named in a carve's
* `sources` picks it up on the next analysis without editing that carve.
*
* Dedupes and preserves first-seen order; an id that resolves to nothing
* (a deleted clip, an empty or vanished group) is dropped rather than kept
* as a dangling reference the analysis would only fail to find anyway.
*/
export function resolveCarveSourceIds(doc: Document, ids: readonly string[]): string[] {
const groupsById = new Map(resolveAudioGroups(doc).map((group) => [group.id, group] as const));
const seen = new Set<string>();
const out: string[] = [];
const add = (id: string): void => {
if (seen.has(id)) return;
seen.add(id);
out.push(id);
};
for (const id of ids) {
const group = groupsById.get(id);
if (group) {
group.memberIds.forEach(add);
} else if (doc.getElementById(id)) {
add(id);
}
}
return out;
}

/** The group a member belongs to, or null. Groups do not nest — this ignores
* `data-audio-group` on an `<hf-audio-group>` element itself.
*
Expand Down
48 changes: 48 additions & 0 deletions packages/lint/src/rules/media.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -500,3 +500,51 @@ describe("audio_volume_double_automation", () => {
expect(res.findings.some((f) => f.code === "audio_volume_double_automation")).toBe(false);
});
});

describe("audio_carve_ungrouped_sources", () => {
const withCarve = (carveJson: string, extra = "") => `<!DOCTYPE html><html><body>
<div id="root" data-composition-id="main" data-start="0" data-width="1920" data-height="1080" data-duration="10">
<audio id="bed" src="bed.wav" data-start="0" data-duration="10" data-fx-carve='${carveJson}'></audio>
${extra}
</div>
</body></html>`;

it("warns when sources names two or more plain clip ids", async () => {
const res = await lintHyperframeHtml(
withCarve(`{"enabled":true,"sources":["vo-1","vo-2"],"strength":0.35}`),
);
const finding = res.findings.find((f) => f.code === "audio_carve_ungrouped_sources");
expect(finding?.severity).toBe("warning");
expect(finding?.elementId).toBe("bed");
});

it("stays quiet when sources names a group", async () => {
const res = await lintHyperframeHtml(
withCarve(
`{"enabled":true,"sources":["voiceover"],"strength":0.35}`,
`<hf-audio-group id="voiceover" data-label="Voiceover"></hf-audio-group>`,
),
);
expect(res.findings.some((f) => f.code === "audio_carve_ungrouped_sources")).toBe(false);
});

it("stays quiet for a single-clip sources list", async () => {
const res = await lintHyperframeHtml(
withCarve(`{"enabled":true,"sources":["narration"],"strength":0.35}`),
);
expect(res.findings.some((f) => f.code === "audio_carve_ungrouped_sources")).toBe(false);
});

it("still warns when one entry is a group and the rest are plain clip ids", async () => {
// Mixing a group with two more bare clip ids is still an ungrouped-source
// rot risk for those two clips — only fully-grouped sources are silent.
const res = await lintHyperframeHtml(
withCarve(
`{"enabled":true,"sources":["voiceover","vo-3","vo-4"],"strength":0.35}`,
`<hf-audio-group id="voiceover" data-label="Voiceover"></hf-audio-group>`,
),
);
const finding = res.findings.find((f) => f.code === "audio_carve_ungrouped_sources");
expect(finding?.severity).toBe("warning");
});
});
48 changes: 48 additions & 0 deletions packages/lint/src/rules/media.ts
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,9 @@ export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> =

// audio_volume_double_automation
findVolumeDoubleAutomationFindings,

// audio_carve_ungrouped_sources
findCarveUngroupedSourcesFindings,
];

/**
Expand Down Expand Up @@ -675,3 +678,48 @@ function findVolumeDoubleAutomationFindings(ctx: LintContext): HyperframeLintFin
}
return findings;
}

/**
* A carve's `sources` naming two or more plain clip ids is the normative
* mistake groups exist to prevent (groups doc §1.6): the list silently rots
* when a voice clip is added or removed, since nothing re-derives it. Naming
* a group instead means membership resolves at analysis time. Silent when
* `sources` already names a group, or names at most one clip.
*/
function findCarveUngroupedSourcesFindings(ctx: LintContext): HyperframeLintFinding[] {
const groupIds = new Set(
ctx.tags.filter((tag) => tag.name === "hf-audio-group").map((tag) => readAttr(tag.raw, "id")),
);

const findings: HyperframeLintFinding[] = [];
for (const tag of ctx.tags) {
const raw = readDecodedAttr(tag.raw, "data-fx-carve");
if (raw === null) continue;
const trimmed = raw.trim();
if (!trimmed.startsWith("{")) continue;
let parsed: unknown;
try {
parsed = JSON.parse(trimmed);
} catch {
continue;
}
const sources = (parsed as { sources?: unknown }).sources;
if (!Array.isArray(sources)) continue;
const clipIds = sources.filter(
(id): id is string => typeof id === "string" && !groupIds.has(id),
);
if (clipIds.length < 2) continue;

const elementId = readAttr(tag.raw, "id") || undefined;
findings.push({
code: "audio_carve_ungrouped_sources",
severity: "warning",
message: `${elementId ? `#${elementId}'s` : "This"} carve names ${clipIds.length} voice clips directly (${clipIds.join(", ")}) instead of a group.`,
elementId,
fixHint:
"Group the voice clips and carve against the group — a hand-rolled clip list silently rots when a clip is added.",
snippet: truncateSnippet(tag.raw),
});
}
return findings;
}
13 changes: 6 additions & 7 deletions packages/studio/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import { useCompositionDimensions } from "./hooks/useCompositionDimensions";
import { useToast } from "./hooks/useToast";
import { useCompositionContentLoader } from "./hooks/useCompositionContentLoader";
import { useStudioUrlState } from "./hooks/useStudioUrlState";
import { useEffectiveTimelineDuration } from "./hooks/useEffectiveTimelineDuration";
import {
buildStudioContextValue,
useGlobalFileDrop,
Expand Down Expand Up @@ -95,13 +96,10 @@ export function StudioApp() {
const setTimelineSelectionSet = usePlayerStore((s) => s.setSelectedElementIds);
const timelineDuration = usePlayerStore((s) => s.duration);
const isPlaying = usePlayerStore((s) => s.isPlaying);
const effectiveTimelineDuration = useMemo(() => {
const maxEnd =
timelineElements.length > 0
? Math.max(...timelineElements.map((el) => el.start + el.duration))
: 0;
return Math.max(timelineDuration, maxEnd);
}, [timelineDuration, timelineElements]);
const effectiveTimelineDuration = useEffectiveTimelineDuration(
timelineDuration,
timelineElements,
);
const { toasts, showToast, dismissToast } = useToast();
const panelLayout = usePanelLayout({
rightCollapsed: initialUrlStateRef.current.rightCollapsed,
Expand Down Expand Up @@ -534,6 +532,7 @@ export function StudioApp() {
domEditSaveTimestampRef={domEditSaveTimestampRef}
recordEdit={editHistory.recordEdit}
onToggleElementHidden={timelineEditing.handleToggleElementHidden}
onAutoGroupCarveSources={timelineEditing.handleAutoGroupCarveSources}
onAddMediaOverlay={handleAddMediaOverlay}
/>
)
Expand Down
67 changes: 7 additions & 60 deletions packages/studio/src/components/StudioRightPanel.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { useCallback, useEffect, useRef, type MutableRefObject } from "react";
import { useCallback, type MutableRefObject } from "react";
import { PropertyPanel } from "./editor/PropertyPanel";
import { LayersPanel } from "./editor/LayersPanel";
import { CaptionPropertyPanel } from "../captions/components/CaptionPropertyPanel";
Expand All @@ -21,18 +21,15 @@ import { usePanelLayoutContext } from "../contexts/PanelLayoutContext";
import { useFileManagerContext } from "../contexts/FileManagerContext";
import { useDomEditContext } from "../contexts/DomEditContext";
import { usePlayerStore } from "../player";
import { waitForMediaJob } from "./studioMediaJobs";
import {
applyColorGradingScopeUpdate,
EMPTY_COLOR_GRADING_SCOPE_RESULT,
type ColorGradingScope,
} from "./studioColorGradingScope";
import type {
AddMediaOverlayHandler,
BackgroundRemovalProgress,
} from "./editor/propertyPanelTypes";
import type { AddMediaOverlayHandler } from "./editor/propertyPanelTypes";
import { timelineKeysForSelections, type ToggleHiddenHandler } from "../utils/studioHelpers";
import { useInspectorSplitResize } from "../hooks/useInspectorSplitResize";
import { useRemoveBackground } from "../hooks/useRemoveBackground";

export interface StudioRightPanelProps extends StudioEditPersistenceProps {
designPanelActive: boolean;
Expand Down Expand Up @@ -70,6 +67,7 @@ export interface StudioRightPanelProps extends StudioEditPersistenceProps {
files: Record<string, { before: string; after: string }>;
}) => Promise<void>;
onToggleElementHidden?: ToggleHiddenHandler;
onAutoGroupCarveSources?: (clipIds: readonly string[], groupId: string) => Promise<void>;
onAddMediaOverlay?: AddMediaOverlayHandler;
}

Expand All @@ -88,6 +86,7 @@ export function StudioRightPanel({
domEditSaveTimestampRef,
recordEdit,
onToggleElementHidden,
onAutoGroupCarveSources,
onAddMediaOverlay,
}: StudioRightPanelProps) {
const {
Expand Down Expand Up @@ -207,14 +206,6 @@ export function StudioRightPanel({
handleInspectorSplitResizeMove,
handleInspectorSplitResizeEnd,
} = useInspectorSplitResize();
const backgroundRemovalAbortRef = useRef<AbortController | null>(null);

useEffect(
() => () => {
backgroundRemovalAbortRef.current?.abort();
},
[],
);

const renderJobs = renderQueue.jobs as RenderJob[];
const inspectorTabActive = rightPanelTab === "design" || rightPanelTab === "layers";
Expand Down Expand Up @@ -281,52 +272,7 @@ export function StudioRightPanel({
],
);

const handleRemoveBackground = useCallback(
// fallow-ignore-next-line complexity
async (
inputPath: string,
options: {
createBackgroundPlate?: boolean;
quality?: "fast" | "balanced" | "best";
onProgress?: (progress: BackgroundRemovalProgress) => void;
},
) => {
const response = await fetch(
`/api/projects/${encodeURIComponent(projectId)}/media/remove-background`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
inputPath,
createBackgroundPlate: options.createBackgroundPlate === true,
quality: options.quality ?? "balanced",
}),
},
);
const data = (await response.json().catch(() => ({}))) as {
jobId?: string;
error?: string;
};
if (!response.ok || !data.jobId) {
throw new Error(data.error || `Background removal failed (${response.status})`);
}
showToast("Removing background...", "info");
backgroundRemovalAbortRef.current?.abort();
const controller = new AbortController();
backgroundRemovalAbortRef.current = controller;
try {
const result = await waitForMediaJob(data.jobId, options.onProgress, controller.signal);
await refreshFileTree();
showToast(`Created transparent asset: ${result.outputPath.split("/").pop()}`, "info");
return result;
} finally {
if (backgroundRemovalAbortRef.current === controller) {
backgroundRemovalAbortRef.current = null;
}
}
},
[projectId, refreshFileTree, showToast],
);
const handleRemoveBackground = useRemoveBackground(projectId, refreshFileTree, showToast);

/**
* A dial being dragged writes to the preview and stops there.
Expand Down Expand Up @@ -372,6 +318,7 @@ export function StudioRightPanel({
copiedAgentPrompt={copiedAgentPrompt}
onClearSelection={clearDomSelection}
onToggleElementHidden={onToggleElementHidden}
onAutoGroupCarveSources={onAutoGroupCarveSources}
onUngroup={handleUngroupSelection}
onSetStyle={handleDomStyleCommit}
onSetAttribute={handleDomAttributeCommit}
Expand Down
2 changes: 2 additions & 0 deletions packages/studio/src/components/editor/PropertyPanelFlat.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ export function PropertyPanelFlat({
onRemoveTextField,
onAskAgent,
onToggleElementHidden,
onAutoGroupCarveSources,
onImportAssets,
onAddMediaOverlay,
onImportFonts,
Expand Down Expand Up @@ -440,6 +441,7 @@ export function PropertyPanelFlat({
element={element}
onSetAttributeQuiet={onSetAttributeQuiet ?? onSetAttributeLive}
onSetAttributeLive={onSetAttributeLive}
onAutoGroupCarveSources={onAutoGroupCarveSources}
/>
),
});
Expand Down
Loading
Loading