diff --git a/README.md b/README.md index 04eb14ea..5b03b959 100644 --- a/README.md +++ b/README.md @@ -285,12 +285,22 @@ a workspace directory containing `scene-manifest.json` with schema Scene** workflow node to select and validate an existing scene directory. Scene-capable generators implement `generate_artifact(input_kind, artifact_path, ...)`; legacy image generators and `POST /generate/from-image` -remain unchanged. The generic `POST /generate/from-artifact` boundary currently -accepts only `scene`, leaving future artifact kinds to separate reviewed changes. +remain unchanged. The generic `POST /generate/from-artifact` boundary accepts +validated `scene` directories and `video` files without converting either to +fake image bytes. For this first contract, `scene` is model-only and must be declared as the single `input` value (not inside `inputs`); process and mixed-input scene nodes are rejected. Model nodes may still accept multiple images and produce a scene. +The dedicated typed-artifact route is used when a model declares exactly +`input: "video"`. Existing video outputs, process video nodes, and `inputs` +arrays containing video remain valid; this feature does not narrow those +extension contracts. Use **Load Video** to import a durable copy under the +workspace before connecting it to a scalar video model input. +The host checks containment, regular-file status, extension, size, and container +signature, while full media decoding remains the extension's responsibility. +Accepted containers are MP4/M4V/MOV, WebM/Matroska, and AVI, up to 8 GiB. + ## Modly CLI diff --git a/api/README.md b/api/README.md index cdcd4d6b..2f30c3e4 100644 --- a/api/README.md +++ b/api/README.md @@ -29,7 +29,7 @@ uvicorn main:app --host 127.0.0.1 --port 8765 --reload | GET | `/model/status` | Model download / load status | | GET | `/model/download` | SSE stream of download progress | | POST | `/generate/from-image` | Start image-to-3D job | -| POST | `/generate/from-artifact` | Start a typed-artifact model job (`scene` only) | +| POST | `/generate/from-artifact` | Start a typed-artifact model job (`scene` or model-input `video`) | | GET | `/generate/status/{job_id}` | Poll job status | ## Model diff --git a/api/routers/generation.py b/api/routers/generation.py index 73d0baae..16a50cbb 100644 --- a/api/routers/generation.py +++ b/api/routers/generation.py @@ -172,7 +172,8 @@ async def generate_from_artifact( raise HTTPException(400, str(exc)) from exc params = {k: v for k, v in request.params.items() if k not in RESERVED_ARTIFACT_PARAMS} - params["scene_manifest_path"] = str(artifact.path) + if artifact.kind == "scene": + params["scene_manifest_path"] = str(artifact.path) collection = sanitize_collection(request.collection) job_id = str(uuid.uuid4()) _purge_old_jobs() @@ -336,11 +337,18 @@ def progress_cb(pct: int, step: str = "") -> None: from services.artifact_input import revalidate_artifact_input model_input = revalidate_artifact_input(registry.WORKSPACE_DIR, model_input) import inspect - supports_cancel = "cancel_event" in inspect.signature(gen.generate_artifact).parameters - output_path = ( - gen.generate_artifact(model_input.kind, model_input.path, params, progress_cb, cancel_event) - if supports_cancel - else gen.generate_artifact(model_input.kind, model_input.path, params, progress_cb) + artifact_parameters = inspect.signature(gen.generate_artifact).parameters + artifact_kwargs = {} + if "cancel_event" in artifact_parameters: + artifact_kwargs["cancel_event"] = cancel_event + if "artifact_snapshot" in artifact_parameters: + artifact_kwargs["artifact_snapshot"] = model_input.snapshot + output_path = gen.generate_artifact( + model_input.kind, + model_input.path, + params, + progress_cb, + **artifact_kwargs, ) else: import inspect diff --git a/api/runner.py b/api/runner.py index 93feef5d..42339034 100644 --- a/api/runner.py +++ b/api/runner.py @@ -169,10 +169,20 @@ def decode_model_input(msg: dict): if "input" not in msg: return base64.b64decode(msg["image_b64"]) value = msg["input"] - if not isinstance(value, dict) or set(value) != {"kind", "path"}: + if not isinstance(value, dict): + raise ValueError("Typed artifact input must contain exactly kind and path") + kind = value.get("kind") + expected_fields = {"kind", "path", "snapshot"} if kind == "video" else {"kind", "path"} + if set(value) != expected_fields: + if kind == "video": + raise ValueError("Video artifact input requires a well-formed snapshot") raise ValueError("Typed artifact input must contain exactly kind and path") from services.artifact_input import TypedArtifactInput, revalidate_artifact_input - typed = TypedArtifactInput(kind=value.get("kind"), path=Path(value.get("path", ""))) + snapshot = None + if kind == "video": + from services.video_input import video_snapshot_from_dict + snapshot = video_snapshot_from_dict(value["snapshot"]) + typed = TypedArtifactInput(kind=kind, path=Path(value.get("path", "")), snapshot=snapshot) return revalidate_artifact_input(WORKSPACE_DIR, typed) @@ -250,9 +260,12 @@ def main() -> None: if not isinstance(params, dict): raise ValueError("Model params must be an object") from services.artifact_input import RESERVED_ARTIFACT_PARAMS - params = {key: value for key, value in params.items() - if key not in RESERVED_ARTIFACT_PARAMS} - params["scene_manifest_path"] = str(model_input.path) + params = { + key: value for key, value in params.items() + if key not in RESERVED_ARTIFACT_PARAMS + } + if model_input.kind == "scene": + params["scene_manifest_path"] = str(model_input.path) if msg.get("outputs_dir"): gen.outputs_dir = Path(msg["outputs_dir"]) gen.outputs_dir.mkdir(parents=True, exist_ok=True) diff --git a/api/schemas/generation.py b/api/schemas/generation.py index 04c18c85..c6b80698 100644 --- a/api/schemas/generation.py +++ b/api/schemas/generation.py @@ -12,8 +12,8 @@ class JobStatus(BaseModel): class GenerateFromArtifactRequest(BaseModel): - """Generic typed-artifact request. Only scene is public in this release.""" - input_kind: Literal["scene"] + """Generic typed-artifact request for model-only scene and video inputs.""" + input_kind: Literal["scene", "video"] input_path: str model_id: str collection: str = "Workflows" diff --git a/api/services/artifact_input.py b/api/services/artifact_input.py index 05f4661d..94f0ec20 100644 --- a/api/services/artifact_input.py +++ b/api/services/artifact_input.py @@ -3,12 +3,14 @@ from pathlib import Path from services.scene_input import validate_scene_input +from services.video_input import VideoSnapshot, validate_video_input -SUPPORTED_ARTIFACT_INPUTS = frozenset({"scene"}) +SUPPORTED_ARTIFACT_INPUTS = frozenset({"scene", "video"}) # Transport parameters set by the host; callers must not be able to forge them. RESERVED_ARTIFACT_PARAMS = frozenset({ "artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path", + "video_path", }) @@ -16,6 +18,7 @@ class TypedArtifactInput: kind: str path: Path + snapshot: VideoSnapshot | None = None def validate_artifact_input(workspace: Path, kind: str, input_path: str) -> TypedArtifactInput: @@ -23,6 +26,9 @@ def validate_artifact_input(workspace: Path, kind: str, input_path: str) -> Type raise ValueError(f"Unsupported artifact input kind: {kind}") if kind == "scene": return TypedArtifactInput(kind="scene", path=validate_scene_input(workspace, input_path)) + if kind == "video": + path, snapshot = validate_video_input(workspace, input_path) + return TypedArtifactInput(kind="video", path=path, snapshot=snapshot) raise ValueError(f"Unsupported artifact input kind: {kind}") @@ -31,4 +37,7 @@ def revalidate_artifact_input(workspace: Path, value: TypedArtifactInput) -> Typ relative = value.path.resolve(strict=True).relative_to(workspace.resolve(strict=True)) except (OSError, ValueError) as exc: raise ValueError("Artifact input is outside the workspace") from exc - return validate_artifact_input(workspace, value.kind, relative.as_posix()) + validated = validate_artifact_input(workspace, value.kind, relative.as_posix()) + if value.kind == "video" and value.snapshot is not None and validated.snapshot != value.snapshot: + raise ValueError("Video input changed after it was queued") + return validated diff --git a/api/services/extension_process.py b/api/services/extension_process.py index 5eaed4b8..a9703e93 100644 --- a/api/services/extension_process.py +++ b/api/services/extension_process.py @@ -18,7 +18,10 @@ import threading import uuid from pathlib import Path -from typing import Callable, Optional +from typing import Callable, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from services.video_input import VideoSnapshot _RUNNER_PATH = Path(__file__).parent.parent / "runner.py" _MISSING_MODULE_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") @@ -404,16 +407,24 @@ def generate_artifact( params: dict, progress_cb: Optional[Callable[[int, str], None]] = None, cancel_event: Optional[threading.Event] = None, + artifact_snapshot: Optional["VideoSnapshot"] = None, ) -> Path: """Send a typed artifact envelope to the isolated runner.""" from services.artifact_input import TypedArtifactInput, revalidate_artifact_input from services.generator_registry import WORKSPACE_DIR + from services.video_input import video_snapshot_to_dict + snapshot_payload = (video_snapshot_to_dict(artifact_snapshot) + if input_kind == "video" else None) validated = revalidate_artifact_input( - WORKSPACE_DIR, TypedArtifactInput(kind=input_kind, path=artifact_path) + WORKSPACE_DIR, + TypedArtifactInput(kind=input_kind, path=artifact_path, snapshot=artifact_snapshot), ) + input_payload = {"kind": validated.kind, "path": str(validated.path)} + if validated.kind == "video": + input_payload["snapshot"] = snapshot_payload return self._generate_request( - {"input": {"kind": validated.kind, "path": str(validated.path)}}, + {"input": input_payload}, params, progress_cb, cancel_event, ) diff --git a/api/services/video_input.py b/api/services/video_input.py new file mode 100644 index 00000000..d5c01e68 --- /dev/null +++ b/api/services/video_input.py @@ -0,0 +1,145 @@ +"""Secure validation for workspace video model inputs.""" +from dataclasses import dataclass +import hashlib +import os +import re +import stat +from pathlib import Path, PurePosixPath, PureWindowsPath + +MAX_VIDEO_BYTES = 8 * 1024**3 +SUPPORTED_VIDEO_EXTENSIONS = frozenset({".mp4", ".m4v", ".mov", ".webm", ".mkv", ".avi"}) +_HEADER_BYTES = 64 + + +@dataclass(frozen=True) +class VideoSnapshot: + size: int + mtime_ns: int + device: int + inode: int + header_sha256: str + + +_SNAPSHOT_FIELDS = frozenset({"size", "mtime_ns", "device", "inode", "header_sha256"}) + + +def video_snapshot_to_dict(snapshot: VideoSnapshot) -> dict: + """Serialize a validated snapshot for the isolated runner envelope.""" + if not isinstance(snapshot, VideoSnapshot): + raise ValueError("Video snapshot is required") + return { + "size": snapshot.size, + "mtime_ns": snapshot.mtime_ns, + "device": snapshot.device, + "inode": snapshot.inode, + "header_sha256": snapshot.header_sha256, + } + + +def video_snapshot_from_dict(value: object) -> VideoSnapshot: + """Strictly reconstruct a snapshot received across the process boundary.""" + if not isinstance(value, dict) or set(value) != _SNAPSHOT_FIELDS: + raise ValueError("Video snapshot must contain exactly the expected fields") + numeric = ("size", "mtime_ns", "device", "inode") + if any(isinstance(value[field], bool) or not isinstance(value[field], int) + for field in numeric): + raise ValueError("Video snapshot numeric fields must be integers") + if value["size"] <= 0 or any(value[field] < 0 for field in numeric[1:]): + raise ValueError("Video snapshot contains invalid numeric values") + digest = value["header_sha256"] + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise ValueError("Video snapshot contains an invalid header digest") + return VideoSnapshot( + size=value["size"], + mtime_ns=value["mtime_ns"], + device=value["device"], + inode=value["inode"], + header_sha256=digest, + ) + + +def _safe_relative(value: str) -> Path: + if not isinstance(value, str) or not value or value != value.strip() or "\x00" in value: + raise ValueError("Video path must be a nonempty workspace-relative path") + normalized = value.replace("\\", "/") + if (PurePosixPath(normalized).is_absolute() or PureWindowsPath(normalized).is_absolute() + or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", normalized) + or re.search(r"%(?:25|2e|2f|5c|00)", normalized, re.I) + or re.search(r"%(?![0-9a-f]{2})", normalized, re.I) + or any(part in ("", ".", "..") for part in normalized.split("/"))): + raise ValueError("Video path must be a safe workspace-relative path") + return Path(*normalized.split("/")) + + +def _reject_link_components(path: Path, root: Path) -> None: + try: + relative = path.relative_to(root) + except ValueError as exc: + raise ValueError("Video path escapes the workspace") from exc + current = root + for part in relative.parts: + current = current / part + try: + info = current.lstat() + except OSError as exc: + raise ValueError("Video file is missing or unreadable") from exc + is_reparse = bool(getattr(info, "st_file_attributes", 0) + & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if current.is_symlink() or is_reparse: + raise ValueError("Video path must not use symlinks or reparse points") + + +def _signature_matches(suffix: str, header: bytes) -> bool: + if suffix in {".mp4", ".m4v", ".mov"}: + return len(header) >= 12 and header[4:8] == b"ftyp" + if suffix in {".webm", ".mkv"}: + return header.startswith(b"\x1a\x45\xdf\xa3") + if suffix == ".avi": + return len(header) >= 12 and header[:4] == b"RIFF" and header[8:12] == b"AVI " + return False + + +def validate_video_input(workspace: Path, video_path: str) -> tuple[Path, VideoSnapshot]: + """Return a canonical file and stable snapshot without decoding the video.""" + root = workspace.resolve(strict=True) + candidate = root / _safe_relative(video_path) + _reject_link_components(candidate, root) + suffix = candidate.suffix.lower() + if suffix not in SUPPORTED_VIDEO_EXTENSIONS: + raise ValueError("Video input uses an unsupported file extension") + + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(candidate, flags) + except OSError as exc: + raise ValueError("Video file is missing or unreadable") from exc + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode): + raise ValueError("Video input must be a regular file") + if info.st_size <= 0: + raise ValueError("Video input must not be empty") + if info.st_size > MAX_VIDEO_BYTES: + raise ValueError("Video input exceeds the 8 GiB size limit") + header = os.read(descriptor, _HEADER_BYTES) + finally: + os.close(descriptor) + + try: + canonical = candidate.resolve(strict=True) + canonical.relative_to(root) + after = candidate.lstat() + except (OSError, ValueError) as exc: + raise ValueError("Video path escapes the workspace") from exc + if not stat.S_ISREG(after.st_mode) or (after.st_dev, after.st_ino) != (info.st_dev, info.st_ino): + raise ValueError("Video file changed while it was being validated") + if not _signature_matches(suffix, header): + raise ValueError("Video file signature does not match its extension") + snapshot = VideoSnapshot( + size=info.st_size, + mtime_ns=info.st_mtime_ns, + device=info.st_dev, + inode=info.st_ino, + header_sha256=hashlib.sha256(header).hexdigest(), + ) + return canonical, snapshot diff --git a/api/tests/test_extension_process.py b/api/tests/test_extension_process.py index 29602cd6..d96faee2 100644 --- a/api/tests/test_extension_process.py +++ b/api/tests/test_extension_process.py @@ -9,6 +9,7 @@ from pathlib import Path from services.extension_process import ExtensionProcess, _venv_python +from services.artifact_input import validate_artifact_input def _make_proc() -> ExtensionProcess: @@ -41,6 +42,27 @@ def test_generate_artifact_sends_typed_scene_without_image_bytes(self) -> None: self.assertEqual(result, manifest) self.assertEqual(calls, [({"input": {"kind": "scene", "path": str(manifest.resolve())}}, {"quality": "high"})]) + def test_generate_artifact_sends_canonical_video_without_image_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + video = workspace / "Workflows" / "clip.mp4" + video.parent.mkdir(parents=True) + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + proc = _make_proc() + calls = [] + proc._generate_request = lambda payload, params, progress, cancel: calls.append(payload) or video + queued = validate_artifact_input(workspace, "video", "Workflows/clip.mp4") + with patch("services.generator_registry.WORKSPACE_DIR", workspace): + proc.generate_artifact("video", video, {}, artifact_snapshot=queued.snapshot) + self.assertEqual(calls[0]["input"]["kind"], "video") + self.assertEqual(calls[0]["input"]["path"], str(video.resolve())) + self.assertEqual(calls[0]["input"]["snapshot"]["size"], len(video.read_bytes())) + + def test_generate_artifact_requires_original_video_snapshot(self) -> None: + proc = _make_proc() + with self.assertRaisesRegex(ValueError, "snapshot"): + proc.generate_artifact("video", Path("clip.mp4"), {}) + def test_read_loop_writes_sentinel_to_own_queue_only(self) -> None: proc = _make_proc() diff --git a/api/tests/test_generator_registry.py b/api/tests/test_generator_registry.py index b21253b7..e48bef78 100644 --- a/api/tests/test_generator_registry.py +++ b/api/tests/test_generator_registry.py @@ -157,7 +157,7 @@ def test_legacy_generator_supports_eager_and_lazy_sibling_imports(self) -> None: self.assertNotIn(str(extension.resolve()), sys.path) def test_scene_and_existing_custom_io_types_are_registered(self) -> None: - for extension_id, input_kind in (("scene-io", "scene"), ("capture-io", "capture"), ("video-io", "video")): + for extension_id, input_kind in (("scene-io", "scene"), ("capture-io", "capture")): extension = self._make_extension(extension_id) manifest = { "id": extension_id, "name": extension_id, "type": "model", @@ -176,7 +176,6 @@ def test_scene_and_existing_custom_io_types_are_registered(self) -> None: self.registry.initialize() self.assertEqual(self.registry.get_manifest("scene-io/generate")["input"], "scene") self.assertEqual(self.registry.get_manifest("capture-io/generate")["input"], "capture") - self.assertEqual(self.registry.get_manifest("video-io/generate")["input"], "video") def test_scene_input_rejects_multi_input_shapes_but_image_multi_can_output_scene(self) -> None: cases = { @@ -203,6 +202,34 @@ def test_scene_input_rejects_multi_input_shapes_but_image_multi_can_output_scene self.assertIn("scene-array/generate", self.registry.load_errors()) self.assertIn("images-scene/generate", self.registry._generators) + def test_existing_video_declarations_remain_discoverable(self) -> None: + cases = { + "video-io": {"input": "video", "output": "mesh"}, + "video-array": {"input": "video", "inputs": ["video"], "output": "mesh"}, + "video-mixed": {"input": "video", "inputs": ["video", "text"], "output": "mesh"}, + "video-output": {"input": "image", "output": "video"}, + } + for extension_id, node in cases.items(): + extension = self._make_extension(extension_id) + (extension / "manifest.json").write_text(json.dumps({ + "id": extension_id, "name": extension_id, "type": "model", + "generator_class": "TestGenerator", + "nodes": [{"id": "generate", **node}], + }), encoding="utf-8") + (extension / "generator.py").write_text( + "from services.generators.base import BaseGenerator\n" + "class TestGenerator(BaseGenerator):\n" + " def load(self): self._model = object()\n" + " def generate(self, value, params, progress_cb=None, cancel_event=None): return self.outputs_dir / 'result.glb'\n", + encoding="utf-8", + ) + + self.registry.initialize() + self.assertEqual(self.registry.get_manifest("video-io/generate")["input"], "video") + self.assertIn("video-array/generate", self.registry._generators) + self.assertIn("video-mixed/generate", self.registry._generators) + self.assertIn("video-output/generate", self.registry._generators) + def test_declared_sources_block_generation_even_when_generator_overrides_readiness(self) -> None: extension = self._make_extension("multi-source") manifest = { diff --git a/api/tests/test_runner.py b/api/tests/test_runner.py index 084e3e7b..54b2aa8b 100644 --- a/api/tests/test_runner.py +++ b/api/tests/test_runner.py @@ -8,6 +8,9 @@ from unittest.mock import patch from contextlib import redirect_stdout from pathlib import Path +from services.generators.base import BaseGenerator +from services.artifact_input import validate_artifact_input +from services.video_input import video_snapshot_to_dict _tmp_ext_dir = tempfile.mkdtemp(prefix="modly-runner-test-") @@ -21,6 +24,15 @@ class RunnerTests(unittest.TestCase): + def test_legacy_generator_fails_actionably_for_video_artifact(self) -> None: + class Legacy(BaseGenerator): + def load(self): pass + def generate(self, image_bytes, params, progress_cb=None, cancel_event=None): return Path("result.glb") + with tempfile.TemporaryDirectory() as tmp: + generator = Legacy(Path(tmp), Path(tmp)) + with self.assertRaisesRegex(NotImplementedError, "does not implement video artifact generation"): + generator.generate_artifact("video", Path(tmp) / "clip.mp4", {}) + def test_decode_typed_scene_revalidates_worker_workspace_and_keeps_legacy_image(self) -> None: with tempfile.TemporaryDirectory() as tmp: workspace = Path(tmp) / "workspace" @@ -45,6 +57,51 @@ def test_runner_model_envelope_rejects_cross_node_dispatch(self) -> None: with self.assertRaisesRegex(ValueError, "does not match"): runner.validate_requested_model({"model_id": "pixal3d/generate"}, manifest, node) + def test_runner_decodes_and_revalidates_video_artifact(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + video = workspace / "Workflows" / "clip.mp4" + video.parent.mkdir(parents=True) + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + queued = validate_artifact_input(workspace, "video", "Workflows/clip.mp4") + with patch.object(runner, "WORKSPACE_DIR", workspace): + typed = runner.decode_model_input({"input": { + "kind": "video", "path": str(video), + "snapshot": video_snapshot_to_dict(queued.snapshot), + }}) + self.assertEqual(typed.kind, "video") + self.assertEqual(typed.path, video.resolve()) + + def test_runner_rejects_swapped_video_using_queued_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + video = workspace / "Workflows" / "clip.mp4" + video.parent.mkdir(parents=True) + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + queued = validate_artifact_input(workspace, "video", "Workflows/clip.mp4") + replacement = video.with_suffix(".replacement") + replacement.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2changed") + os.replace(replacement, video) + with patch.object(runner, "WORKSPACE_DIR", workspace), self.assertRaisesRegex(ValueError, "changed"): + runner.decode_model_input({"input": { + "kind": "video", "path": str(video), + "snapshot": video_snapshot_to_dict(queued.snapshot), + }}) + + def test_runner_requires_well_formed_video_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + video = workspace / "Workflows" / "clip.mp4" + video.parent.mkdir(parents=True) + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + for input_value in ( + {"kind": "video", "path": str(video)}, + {"kind": "video", "path": str(video), "snapshot": {}}, + {"kind": "video", "path": str(video), "snapshot": {"size": True}}, + ): + with self.subTest(input_value=input_value), patch.object(runner, "WORKSPACE_DIR", workspace), self.assertRaisesRegex(ValueError, "snapshot"): + runner.decode_model_input({"input": input_value}) + def test_select_node_uses_model_dir_override(self) -> None: manifest = { "nodes": [ diff --git a/api/tests/test_scene_generation.py b/api/tests/test_scene_generation.py index 90333ca5..cbbf3781 100644 --- a/api/tests/test_scene_generation.py +++ b/api/tests/test_scene_generation.py @@ -13,6 +13,7 @@ import routers.generation as generation import services.generator_registry as registry from schemas.generation import GenerateFromArtifactRequest +from services.artifact_input import validate_artifact_input class _Registry: @@ -58,7 +59,7 @@ def test_generic_route_queues_typed_scene_and_strips_reserved_params(self): self.assertFalse(self.registry.switched) def test_generic_route_rejects_unsupported_kind_and_model_mismatch(self): - for kind in ("video", "capture", "image"): + for kind in ("capture", "image"): with self.subTest(kind=kind), self.assertRaises(ValidationError): GenerateFromArtifactRequest( input_kind=kind, input_path="Workflows/room", model_id="demo/scene") @@ -68,6 +69,22 @@ def test_generic_route_rejects_unsupported_kind_and_model_mismatch(self): input_kind="scene", input_path="Workflows/room", model_id="demo/image"), BackgroundTasks())) self.assertEqual(caught.exception.status_code, 400) + def test_generic_route_queues_video_without_forgeable_transport_params(self): + video = self.workspace / "Workflows" / "clip.mp4" + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + self.registry.get_manifest = lambda _model_id: {"input": "video", "output": "mesh"} + tasks = BackgroundTasks() + asyncio.run(generation.generate_from_artifact(GenerateFromArtifactRequest( + input_kind="video", input_path="Workflows/clip.mp4", model_id="demo/video", + params={"video_path": "/etc/passwd", "input_path": "fake", "quality": "high"}, + ), tasks)) + queued = tasks.tasks[0] + self.assertEqual(queued.args[1].kind, "video") + self.assertEqual(queued.args[1].path, video.resolve()) + self.assertNotIn("video_path", queued.args[2]) + self.assertNotIn("input_path", queued.args[2]) + self.assertEqual(queued.args[5], "demo/video") + def test_rejects_traversal_before_switch_or_queue(self): with self.assertRaises(HTTPException): asyncio.run(generation.generate_from_artifact(GenerateFromArtifactRequest( @@ -124,6 +141,41 @@ def test_missing_pinned_model_fails_actionably(self): self.assertEqual(generation._jobs[job_id].status, "error") self.assertIn("Unknown model ID: demo/missing", generation._jobs[job_id].error) + def test_video_job_stays_pinned_and_receives_snapshot_and_cancellation_event(self): + video = self.workspace / "Workflows" / "clip.mp4" + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + artifact = validate_artifact_input(self.workspace, "video", "Workflows/clip.mp4") + calls = [] + + class Generator: + outputs_dir = None + def is_loaded(self): return True + def generate_artifact(self, kind, path, params, progress_cb, cancel_event=None, + artifact_snapshot=None): + calls.append((kind, path, cancel_event, artifact_snapshot)) + output = Path(self.outputs_dir) / "result.glb" + output.write_bytes(b"glb") + return output + + generator = Generator() + registry_stub = type("Registry", (), { + "assert_weight_variant_installed": lambda self, params, model_id=None: None, + "model_status": lambda self, model_id: {"name": model_id, "downloaded": True, "loaded": True}, + "get_generator": lambda self, model_id: generator if model_id == "demo/video" else None, + "activate_ready_generator": lambda self, model_id: generator if model_id == "demo/video" else None, + "get_active": lambda self: (_ for _ in ()).throw(AssertionError("active model must not be used")), + })() + job_id = "pinned-video" + generation._jobs[job_id] = generation.JobStatus(job_id=job_id, status="pending", progress=0) + generation._cancel_events[job_id] = threading.Event() + with patch.object(generation, "generator_registry", registry_stub): + asyncio.run(generation._run_generation( + job_id, artifact, {}, "Workflows", "mesh", "demo/video", + )) + self.assertEqual(calls[0][:2], ("video", video.resolve())) + self.assertIs(calls[0][2], generation._cancel_events[job_id]) + self.assertEqual(calls[0][3], artifact.snapshot) + def test_interleaved_pinned_jobs_serialize_model_lifecycle_and_cancel_exact_job(self): class Proc: def __init__(self, owner): diff --git a/api/tests/test_video_input.py b/api/tests/test_video_input.py new file mode 100644 index 00000000..784e0068 --- /dev/null +++ b/api/tests/test_video_input.py @@ -0,0 +1,67 @@ +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from services.artifact_input import revalidate_artifact_input, validate_artifact_input +from services.video_input import validate_video_input + + +MP4 = b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2" + + +class VideoInputTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self.tmp.name) / "workspace" + self.video = self.workspace / "Workflows" / "Videos" / "clip.mp4" + self.video.parent.mkdir(parents=True) + self.video.write_bytes(MP4) + + def tearDown(self): + self.tmp.cleanup() + + def test_accepts_supported_video_signatures_as_canonical_regular_files(self): + samples = { + "clip.mp4": MP4, + "clip.mov": b"\x00\x00\x00\x14ftypqt \x00\x00\x00\x00", + "clip.webm": b"\x1aE\xdf\xa3\x9fB\x86\x81\x01", + "clip.mkv": b"\x1aE\xdf\xa3\x9fB\x86\x81\x01", + "clip.avi": b"RIFF\x10\x00\x00\x00AVI LIST", + } + for name, payload in samples.items(): + path = self.video.parent / name + path.write_bytes(payload) + with self.subTest(name=name): + self.assertEqual(validate_video_input(self.workspace, f"Workflows/Videos/{name}")[0], path.resolve()) + + def test_rejects_traversal_absolute_encoded_symlink_and_non_regular_paths(self): + outside = Path(self.tmp.name) / "outside.mp4" + outside.write_bytes(MP4) + (self.video.parent / "link.mp4").symlink_to(outside) + for value in ("../outside.mp4", "/etc/passwd", "C:/outside.mp4", "Workflows/%2e%2e/out.mp4", "Workflows/Videos/link.mp4", "Workflows/Videos"): + with self.subTest(value=value), self.assertRaises(ValueError): + validate_video_input(self.workspace, value) + + def test_rejects_extension_magic_mismatch_empty_and_oversized_files(self): + bad = self.video.parent / "bad.mp4" + bad.write_bytes(b"not a video") + empty = self.video.parent / "empty.webm" + empty.write_bytes(b"") + unsupported = self.video.parent / "clip.exe" + unsupported.write_bytes(MP4) + for value in (bad, empty, unsupported): + with self.subTest(value=value.name), self.assertRaises(ValueError): + validate_video_input(self.workspace, value.relative_to(self.workspace).as_posix()) + with patch("services.video_input.MAX_VIDEO_BYTES", len(MP4) - 1): + with self.assertRaisesRegex(ValueError, "size limit"): + validate_video_input(self.workspace, "Workflows/Videos/clip.mp4") + + def test_snapshot_detects_replacement_before_inference(self): + artifact = validate_artifact_input(self.workspace, "video", "Workflows/Videos/clip.mp4") + replacement = self.video.with_suffix(".replacement") + replacement.write_bytes(MP4 + b"changed") + os.replace(replacement, self.video) + with self.assertRaisesRegex(ValueError, "changed"): + revalidate_artifact_input(self.workspace, artifact) diff --git a/electron/main/extension-install-utils.test.mjs b/electron/main/extension-install-utils.test.mjs index ec192d71..1b3d1b94 100644 --- a/electron/main/extension-install-utils.test.mjs +++ b/electron/main/extension-install-utils.test.mjs @@ -136,6 +136,25 @@ test('scene is model-only, single-input, while image-multi to scene stays valid' } }) +test('video declarations remain backward compatible across model and process nodes', () => { + const mod = loadModule() + const modelFiles = { hasEntryFile: () => false, hasGeneratorFile: () => true } + const processFiles = { hasEntryFile: () => true, hasGeneratorFile: () => false } + for (const node of [ + { id: 'scalar', input: 'video', output: 'mesh' }, + { id: 'array', input: 'video', inputs: ['video'], output: 'mesh' }, + { id: 'mixed', input: 'video', inputs: ['video', 'text'], output: 'mesh' }, + { id: 'hidden', input: 'image', inputs: ['video'], output: 'mesh' }, + { id: 'output', input: 'image', output: 'video' }, + ]) { + assert.doesNotThrow(() => mod.validateInstallManifest({ id: 'model', generator_class: 'Generator', nodes: [node] }, modelFiles, 'repository')) + } + assert.doesNotThrow(() => mod.validateInstallManifest({ + id: 'process', type: 'process', entry: 'processor.js', + nodes: [{ id: 'run', input: 'video', inputs: ['video', 'text'], output: 'video' }], + }, processFiles, 'repository')) +}) + test('validateInstallManifest rejects malformed or process model_sources', () => { const mod = loadModule() const source = { @@ -506,4 +525,3 @@ test('validateInstallManifest validates weight variants and keeps them off proce nodes: [{ id: 'run', hf_repo: 'org/model', weight_variants: weightVariants }], }, files, 'repository'), /weight_variants is supported only for model nodes/) }) - diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index e9cac040..6d956fda 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -9,6 +9,7 @@ import * as tar from 'tar' import * as os from 'os' import { promisify } from 'util' import { PythonBridge, API_BASE_URL } from './python-bridge' +import { importVideoToWorkspace } from './video-import' import { isModelDownloaded, listDownloadedModels, @@ -360,6 +361,19 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe return result.canceled ? null : result.filePaths[0] }) + ipcMain.handle('fs:selectVideo', async () => { + const win = getWindow() + if (!win) return null + const result = await dialog.showOpenDialog(win, { + title: 'Import a video', + filters: [{ name: 'Videos', extensions: ['mp4', 'm4v', 'mov', 'webm', 'mkv', 'avi'] }], + properties: ['openFile'], + }) + if (result.canceled) return null + const workspaceDir = getSettings(app.getPath('userData')).workspaceDir + return importVideoToWorkspace(result.filePaths[0], workspaceDir) + }) + ipcMain.handle('fs:selectMeshFile', async () => { const win = getWindow() if (!win) return null diff --git a/electron/main/video-import.test.mjs b/electron/main/video-import.test.mjs new file mode 100644 index 00000000..2b619f94 --- /dev/null +++ b/electron/main/video-import.test.mjs @@ -0,0 +1,24 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, readFileSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const outfile = join(mkdtempSync(join(tmpdir(), 'modly-video-import-build-')), 'import.cjs') +writeFileSync(outfile, buildSync({ entryPoints: [resolve('electron/main/video-import.ts')], bundle: true, platform: 'node', format: 'cjs', write: false }).outputFiles[0].text) +const { importVideoToWorkspace } = createRequire(import.meta.url)(outfile) + +test('video import creates a durable workspace copy and refuses symlink sources', async () => { + const root = mkdtempSync(join(tmpdir(), 'modly-video-import-')) + const source = join(root, 'source clip.mp4') + const payload = Buffer.from('\x00\x00\x00\x18ftypisom') + writeFileSync(source, payload) + const result = await importVideoToWorkspace(source, join(root, 'workspace')) + assert.match(result.workspacePath, /^Workflows\/Imported Videos\/[0-9a-f-]+-source_clip\.mp4$/) + assert.deepEqual(readFileSync(result.absolutePath), payload) + const link = join(root, 'linked.mp4') + symlinkSync(source, link) + await assert.rejects(importVideoToWorkspace(link, join(root, 'workspace')), /regular file/) +}) diff --git a/electron/main/video-import.ts b/electron/main/video-import.ts new file mode 100644 index 00000000..fbf9c800 --- /dev/null +++ b/electron/main/video-import.ts @@ -0,0 +1,29 @@ +import { randomUUID } from 'node:crypto' +import { copyFile, lstat, mkdir } from 'node:fs/promises' +import { basename, extname, join } from 'node:path' + +const MAX_VIDEO_BYTES = 8 * 1024 ** 3 +const VIDEO_EXTENSIONS = new Set(['.mp4', '.m4v', '.mov', '.webm', '.mkv', '.avi']) + +export interface ImportedWorkspaceVideo { + workspacePath: string + absolutePath: string +} + +export async function importVideoToWorkspace(source: string, workspaceDir: string): Promise { + const info = await lstat(source) + if (!info.isFile() || info.isSymbolicLink()) throw new Error('Selected video must be a regular file') + if (info.size <= 0 || info.size > MAX_VIDEO_BYTES) throw new Error('Selected video must be between 1 byte and 8 GiB') + const extension = extname(source).toLowerCase() + if (!VIDEO_EXTENSIONS.has(extension)) throw new Error('Selected video uses an unsupported extension') + + const importDir = join(workspaceDir, 'Workflows', 'Imported Videos') + await mkdir(importDir, { recursive: true }) + const safeName = basename(source).replace(/[^a-zA-Z0-9._-]+/g, '_') + const fileName = `${randomUUID()}-${safeName}` + const absolutePath = join(importDir, fileName) + await copyFile(source, absolutePath) + const copied = await lstat(absolutePath) + if (!copied.isFile() || copied.size !== info.size) throw new Error('Imported video copy could not be verified') + return { workspacePath: `Workflows/Imported Videos/${fileName}`, absolutePath } +} diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index 33944a57..ae9cf7be 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -79,6 +79,8 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra webUtils.getPathForFile(file), selectImage: (): Promise => ipcRenderer.invoke('fs:selectImage') as Promise, + selectVideo: (): Promise<{ workspacePath: string; absolutePath: string } | null> => + ipcRenderer.invoke('fs:selectVideo') as Promise<{ workspacePath: string; absolutePath: string } | null>, selectMeshFile: (): Promise => ipcRenderer.invoke('fs:selectMeshFile') as Promise, saveModel: (defaultName: string): Promise => diff --git a/src/areas/workflows/WorkflowsPage.tsx b/src/areas/workflows/WorkflowsPage.tsx index 8fc4cea6..7eff2fc0 100644 --- a/src/areas/workflows/WorkflowsPage.tsx +++ b/src/areas/workflows/WorkflowsPage.tsx @@ -28,6 +28,7 @@ import TextNode from './nodes/TextNode' import AddToSceneNode from './nodes/AddToSceneNode' import Load3DMeshNode from './nodes/Load3DMeshNode' import LoadSceneNode from './nodes/LoadSceneNode' +import LoadVideoNode from './nodes/LoadVideoNode' import PreviewImageNode from './nodes/PreviewImageNode' import ImagePreviewNode from './nodes/ImagePreviewNode' import WaitNode from './nodes/WaitNode' @@ -39,7 +40,7 @@ import WorkflowEdge from './nodes/WorkflowEdge' const DRAG_KEY = 'modly/extension-id' const DRAG_NODE_KEY = 'modly/node-type' -const NODE_TYPES = { extensionNode: ExtensionNode, imageNode: ImageNode, textNode: TextNode, outputNode: AddToSceneNode, meshNode: Load3DMeshNode, sceneNode: LoadSceneNode, previewNode: PreviewImageNode, imagePreviewNode: ImagePreviewNode, waitNode: WaitNode, whileNode: WhileNode, forEachNode: ForEachNode } +const NODE_TYPES = { extensionNode: ExtensionNode, imageNode: ImageNode, textNode: TextNode, outputNode: AddToSceneNode, meshNode: Load3DMeshNode, sceneNode: LoadSceneNode, videoNode: LoadVideoNode, previewNode: PreviewImageNode, imagePreviewNode: ImagePreviewNode, waitNode: WaitNode, whileNode: WhileNode, forEachNode: ForEachNode } // Loop-container node types: resizable frames whose children form a loop body. // (For Each iterators are plain source nodes, not containers.) @@ -63,15 +64,16 @@ function findWhileContainerAt(nodes: Node[], pos: { x: number; y: number }): Nod // ─── IO badge ───────────────────────────────────────────────────────────────── -const IO_STYLES: Record<'image' | 'text' | 'mesh' | 'audio' | 'scene', string> = { +const IO_STYLES: Record<'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video', string> = { audio: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/25', image: 'bg-sky-500/15 text-sky-400 border-sky-500/25', mesh: 'bg-violet-500/15 text-violet-400 border-violet-500/25', text: 'bg-amber-500/15 text-amber-400 border-amber-500/25', - scene: 'bg-pink-500/15 text-pink-400 border-pink-500/25', + scene: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/25', + video: 'bg-pink-500/15 text-pink-400 border-pink-500/25', } -function IoBadge({ type }: { type: 'image' | 'text' | 'mesh' | 'audio' | 'scene' }) { +function IoBadge({ type }: { type: 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' }) { return ( {type} @@ -101,7 +103,8 @@ const PANEL_BUILTIN_NODES = [ { type: 'imageNode', label: 'Image', color: '#38bdf8', icon: <> }, { type: 'textNode', label: 'Text', color: '#fbbf24', icon: <> }, { type: 'meshNode', label: 'Load 3D Mesh', color: '#a78bfa', icon: <> }, - { type: 'sceneNode', label: 'Load Scene', color: '#f472b6', icon: <> }, + { type: 'sceneNode', label: 'Load Scene', color: '#34d399', icon: <> }, + { type: 'videoNode', label: 'Load Video', color: '#f472b6', icon: <> }, { type: 'outputNode', label: 'Add to Scene', color: '#a78bfa', icon: <> }, { type: 'previewNode', label: 'Preview Views', color: '#38bdf8', icon: <> }, { type: 'imagePreviewNode', label: 'Preview Image', color: '#38bdf8', icon: <> }, @@ -349,7 +352,8 @@ const BUILTIN_NODES = [ { type: 'imageNode', label: 'Image', color: '#38bdf8', description: 'Image input' }, { type: 'textNode', label: 'Text', color: '#fbbf24', description: 'Text input' }, { type: 'meshNode', label: 'Load 3D Mesh', color: '#a78bfa', description: 'Load a 3D mesh file or use current model' }, - { type: 'sceneNode', label: 'Load Scene', color: '#f472b6', description: 'Load and validate a workspace scene directory' }, + { type: 'sceneNode', label: 'Load Scene', color: '#34d399', description: 'Load and validate a workspace scene directory' }, + { type: 'videoNode', label: 'Load Video', color: '#f472b6', description: 'Import a durable workspace video file' }, { type: 'outputNode', label: 'Add to Scene', color: '#a78bfa', description: 'Output node — adds the mesh to the 3D scene' }, { type: 'previewNode', label: 'Preview Views', color: '#38bdf8', description: 'Displays multi-view image outputs in a 2×3 grid' }, { type: 'imagePreviewNode', label: 'Preview Image', color: '#38bdf8', description: 'Displays a single image output in the workflow' }, @@ -708,6 +712,7 @@ function getNodeOutputType(node: Node | undefined, allExts: WorkflowExtension[]) if (node.type === 'imageNode') return 'image' if (node.type === 'meshNode') return 'mesh' if (node.type === 'sceneNode') return 'scene' + if (node.type === 'videoNode') return 'video' if (node.type === 'textNode') return 'text' if (node.type === 'imagePreviewNode') return 'image' return allExts.find((e) => e.id === (node.data as WFNodeData)?.extensionId)?.output @@ -1379,7 +1384,8 @@ const MINI_NODE_TINTS: Record = { imageNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, textNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, meshNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, - sceneNode: { fill: 'rgba(244,114,182,0.22)', stroke: '#f472b6' }, + sceneNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, + videoNode: { fill: 'rgba(244,114,182,0.22)', stroke: '#f472b6' }, extensionNode: { fill: 'rgba(167,139,250,0.24)', stroke: '#a78bfa' }, outputNode: { fill: 'rgba(56,189,248,0.22)', stroke: '#38bdf8' }, previewNode: { fill: 'rgba(56,189,248,0.22)', stroke: '#38bdf8' }, diff --git a/src/areas/workflows/mockExtensions.ts b/src/areas/workflows/mockExtensions.ts index 9bdb9a0a..4edff362 100644 --- a/src/areas/workflows/mockExtensions.ts +++ b/src/areas/workflows/mockExtensions.ts @@ -10,10 +10,10 @@ export interface WorkflowExtension { nodeId: string // "node_id" name: string description: string - input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' - inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene')[] // multi-input; overrides input when set + input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' + inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video')[] // multi-input; overrides input when set inputLabels?: string[] // display labels per input slot - output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' + output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' params: ParamSchema[] builtin: boolean type: 'model' | 'process' diff --git a/src/areas/workflows/nodes/ExtensionNode.tsx b/src/areas/workflows/nodes/ExtensionNode.tsx index 17e5f3fa..40229e99 100644 --- a/src/areas/workflows/nodes/ExtensionNode.tsx +++ b/src/areas/workflows/nodes/ExtensionNode.tsx @@ -18,7 +18,8 @@ const HANDLE_COLOR: Record = { image: '#38bdf8', mesh: '#a78bfa', text: '#fbbf24', - scene: '#f472b6', + scene: '#34d399', + video: '#f472b6', } const TAG_CLS: Record = { @@ -26,7 +27,8 @@ const TAG_CLS: Record = { image: 'border-sky-500/30 bg-sky-500/10 text-sky-400', mesh: 'border-violet-500/30 bg-violet-500/10 text-violet-400', text: 'border-amber-500/30 bg-amber-500/10 text-amber-400', - scene: 'border-pink-500/30 bg-pink-500/10 text-pink-400', + scene: 'border-emerald-500/30 bg-emerald-500/10 text-emerald-400', + video: 'border-pink-500/30 bg-pink-500/10 text-pink-400', } // ─── Param control ──────────────────────────────────────────────────────────── diff --git a/src/areas/workflows/nodes/LoadVideoNode.tsx b/src/areas/workflows/nodes/LoadVideoNode.tsx new file mode 100644 index 00000000..4a52046f --- /dev/null +++ b/src/areas/workflows/nodes/LoadVideoNode.tsx @@ -0,0 +1,46 @@ +import { useCallback, useLayoutEffect, useRef, useState } from 'react' +import { Handle, Position, useReactFlow } from '@xyflow/react' +import type { WFNodeData } from '@shared/types/electron.d' + +import BaseNode from './BaseNode' +import { normalizeVideoSource } from '../workflowVideoSource' + +const OUTPUT_COLOR = '#f472b6' + +export default function LoadVideoNode({ id, data, selected }: { id: string; data: WFNodeData; selected?: boolean }) { + const { updateNodeData } = useReactFlow() + const ioRowRef = useRef(null) + const [handleTop, setHandleTop] = useState('50%') + useLayoutEffect(() => { + if (ioRowRef.current) setHandleTop(`${ioRowRef.current.offsetTop + ioRowRef.current.offsetHeight / 2}px`) + }, []) + + const workspacePath = typeof data.params.workspacePath === 'string' ? data.params.workspacePath : '' + const error = typeof data.params.error === 'string' ? data.params.error : undefined + + const browse = useCallback(async () => { + const selectedVideo = await window.electron.fs.selectVideo() + if (!selectedVideo) return + const settings = await window.electron.settings.get() + const normalized = normalizeVideoSource(selectedVideo.workspacePath, settings.workspaceDir) + updateNodeData(id, { params: normalized + ? { ...data.params, workspacePath: normalized.workspacePath, error: undefined } + : { ...data.params, workspacePath: undefined, absolutePath: undefined, error: 'Video import did not return a safe workspace path.' } }) + }, [id, data, updateNodeData]) + + return ( + } + subheader={
video
} + handles={} + > +
+ +
+ {workspacePath || 'Imports a durable copy into the workspace for downstream video model nodes.'} +
+ {error &&
{error}
} +
+
+ ) +} diff --git a/src/areas/workflows/nodes/WorkflowEdge.tsx b/src/areas/workflows/nodes/WorkflowEdge.tsx index 90d32931..3f37ff57 100644 --- a/src/areas/workflows/nodes/WorkflowEdge.tsx +++ b/src/areas/workflows/nodes/WorkflowEdge.tsx @@ -8,6 +8,8 @@ const HANDLE_COLOR: Record = { image: '#38bdf8', mesh: '#a78bfa', text: '#fbbf24', + scene: '#34d399', + video: '#f472b6', } export default function WorkflowEdge({ @@ -33,6 +35,10 @@ export default function WorkflowEdge({ ? HANDLE_COLOR.text : sourceNode?.type === 'meshNode' ? HANDLE_COLOR.mesh + : sourceNode?.type === 'sceneNode' + ? HANDLE_COLOR.scene + : sourceNode?.type === 'videoNode' + ? HANDLE_COLOR.video : (HANDLE_COLOR[allExtensions.find((e) => e.id === sourceNode?.data?.extensionId)?.output ?? ''] ?? '#52525b') // For multi-input nodes pick the color of the specific connected handle diff --git a/src/areas/workflows/preflight.test.mjs b/src/areas/workflows/preflight.test.mjs index 3b4df8aa..659bfc73 100644 --- a/src/areas/workflows/preflight.test.mjs +++ b/src/areas/workflows/preflight.test.mjs @@ -166,3 +166,22 @@ test('renderer fails closed for unsupported process and mixed scene node shapes' assert.ok(issues.some((issue) => issue.key === 'target:unsupported-scene-shape')) } }) + +test('validated video sources preserve model, process, array, and video-output contracts', () => { + const { validateWorkflowPreflight } = loadModule() + const video = { id: 'video', type: 'videoNode', position: { x: 0, y: 0 }, data: { params: { workspacePath: 'Workflows/Videos/clip.mp4' } } } + const target = { id: 'target', type: 'extensionNode', position: { x: 0, y: 0 }, data: { extensionId: 'pack/process-node' } } + for (const extension of [ + ext({ input: 'video', output: 'mesh', type: 'model' }), + ext({ input: 'video', output: 'mesh', type: 'process' }), + ext({ input: 'video', inputs: ['video'], output: 'mesh', type: 'model' }), + ext({ input: 'video', output: 'video', type: 'model' }), + ]) { + const issues = validateWorkflowPreflight(wf([video, target], [{ id: 'e', source: 'video', target: 'target' }]), [extension]) + assert.deepEqual(issues, []) + } + for (const params of [{ path: '/tmp/clip.mp4' }, { workspacePath: '../clip.mp4' }, { workspacePath: 'Workflows/clip.exe' }]) { + const invalid = { ...video, data: { params } } + assert.ok(validateWorkflowPreflight(wf([invalid], []), []).some((issue) => issue.key === 'video:video-invalid')) + } +}) diff --git a/src/areas/workflows/preflight.ts b/src/areas/workflows/preflight.ts index 068a9277..5d0e30f0 100644 --- a/src/areas/workflows/preflight.ts +++ b/src/areas/workflows/preflight.ts @@ -2,8 +2,9 @@ import type { Workflow, WFNode } from '@shared/types/electron.d' import { getWorkflowExtension, type WorkflowExtension } from './mockExtensions' import { hasUnsupportedSceneShape } from './sceneShape' import { isPassthrough, isBranchConsumer, resolveDataSource, nearestUpstreamWaits } from './nodeBehaviors' +import { normalizeVideoSource } from './workflowVideoSource' -type DataType = 'image' | 'text' | 'mesh' | 'audio' | 'scene' +type DataType = 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' export interface WorkflowPreflightIssue { key: string @@ -16,6 +17,7 @@ function nodeLabel(node: WFNode, allExtensions: WorkflowExtension[]): string { if (node.type === 'textNode') return 'Text' if (node.type === 'meshNode') return 'Load 3D Mesh' if (node.type === 'sceneNode') return 'Load Scene' + if (node.type === 'videoNode') return 'Load Video' if (node.type === 'outputNode') return 'Add to Scene' if (node.type === 'previewNode') return 'Preview Views' if (node.type === 'imagePreviewNode') return 'Preview Image' @@ -31,6 +33,7 @@ function nodeLabel(node: WFNode, allExtensions: WorkflowExtension[]): string { function formatType(type: DataType): string { if (type === 'scene') return 'scene' + if (type === 'video') return 'video' if (type === 'mesh') return 'mesh' if (type === 'image') return 'image' if (type === 'audio') return 'audio' @@ -48,6 +51,7 @@ function getNodeOutputType(node: WFNode, allExtensions: WorkflowExtension[]): Da if (node.type === 'textNode') return 'text' if (node.type === 'meshNode' || node.type === 'outputNode') return 'mesh' if (node.type === 'sceneNode') return 'scene' + if (node.type === 'videoNode') return 'video' if (node.type === 'previewNode') return 'image' if (node.type === 'imagePreviewNode') return 'image' if (node.type === 'forEachNode') { @@ -106,6 +110,14 @@ export function validateWorkflowPreflight( message: 'Load Scene needs a validated scene directory.', }) } + if (node.type === 'videoNode' && !normalizeVideoSource( + node.data.params?.workspacePath as string | undefined, '/workspace', + )) { + pushIssue(issues, { + key: `${node.id}:video-invalid`, nodeId: node.id, + message: 'Load Video needs an imported workspace video file.', + }) + } // A node fed by two different Wait branches can't be scheduled into a single // branch — it would run before either branch produces its mesh. @@ -140,7 +152,6 @@ export function validateWorkflowPreflight( }) continue } - const incomingEdges = workflow.edges.filter((edge) => edge.target === node.id) const requiredTypes = [...new Set((ext.inputs ?? [ext.input]) as DataType[])] diff --git a/src/areas/workflows/slotInputs.test.mjs b/src/areas/workflows/slotInputs.test.mjs index 16adecd2..d9b76990 100644 --- a/src/areas/workflows/slotInputs.test.mjs +++ b/src/areas/workflows/slotInputs.test.mjs @@ -46,6 +46,13 @@ test('an audio slot becomes the primary path', () => { assert.deepEqual(r.extraImagePaths, []) }) +test('a video slot in an inputs array becomes the primary path', () => { + const r = assignSlotFilePaths(['video', 'text'], ['clip.mp4', undefined]) + assert.equal(r.nodeInputPath, 'clip.mp4') + assert.equal(r.nodeInputMeshPath, undefined) + assert.deepEqual(r.extraImagePaths, []) +}) + test('text slots never claim a file path, and empty slots are skipped', () => { const r = assignSlotFilePaths(['text', 'image'], ['leaked.png', undefined]) assert.equal(r.nodeInputPath, undefined) diff --git a/src/areas/workflows/slotInputs.ts b/src/areas/workflows/slotInputs.ts index c07705e9..4219a5e0 100644 --- a/src/areas/workflows/slotInputs.ts +++ b/src/areas/workflows/slotInputs.ts @@ -4,7 +4,7 @@ // the file path resolved for each slot, indexed the same way (undefined where the // slot carries text or nothing). Pure so it can be tested without the store. -export type SlotInputType = 'image' | 'text' | 'mesh' | 'audio' +export type SlotInputType = 'image' | 'text' | 'mesh' | 'audio' | 'video' export interface SlotFilePaths { /** Primary file: what the extension receives as `filePath` when no mesh is present. */ @@ -28,7 +28,7 @@ export function assignSlotFilePaths( } else if (inputTypes[i] === 'image') { if (!out.nodeInputPath) out.nodeInputPath = fp else out.extraImagePaths.push(fp) - } else if (inputTypes[i] === 'audio') { + } else if (inputTypes[i] === 'audio' || inputTypes[i] === 'video') { if (!out.nodeInputPath) out.nodeInputPath = fp } } diff --git a/src/areas/workflows/workflowRunStore.ts b/src/areas/workflows/workflowRunStore.ts index 1c64c0c5..35e211df 100644 --- a/src/areas/workflows/workflowRunStore.ts +++ b/src/areas/workflows/workflowRunStore.ts @@ -9,6 +9,7 @@ import type { Workflow, WFNode, WFEdge } from '@shared/types/electron.d' import { isBranchStarter, isSceneOutput, resolveDataSource, reachesSceneOutput, nearestUpstreamWaits } from './nodeBehaviors' import { assignSlotFilePaths } from './slotInputs' import type { SlotInputType } from './slotInputs' +import { normalizeVideoSource } from './workflowVideoSource' // ─── Types ──────────────────────────────────────────────────────────────────── @@ -324,6 +325,7 @@ async function executeExtensionNode( let nodeInputText: string | undefined let nodeInputMeshPath: string | undefined let nodeInputScenePath: string | undefined + let nodeInputVideoPath: string | undefined // Per-slot texts for multi-text-input nodes (e.g. positive/negative prompts). // Indexed by target handle: input-0 → texts[0], input-1 → texts[1]. const nodeInputTexts: (string | undefined)[] = [] @@ -337,7 +339,9 @@ async function executeExtensionNode( // Scene is intentionally a single-input-only model contract. The guard // above rejects it before this multi-slot path, and filtering it here also // narrows the remaining declarations to assignSlotFilePaths' exact ABI. - const inputTypes: SlotInputType[] = ext.inputs.filter((input) => input !== 'scene') + const inputTypes = ext.inputs.filter( + (input): input is SlotInputType => input !== 'scene', + ) // Resolved by target handle first, then typed by that slot's declared input -- // not by the arrival order of `incomingEdges`, which does not match slot order. const inputPaths = new Array(inputTypes.length).fill(undefined) @@ -364,6 +368,7 @@ async function executeExtensionNode( if (src?.filePath !== undefined) nodeInputPath = src.filePath if (src?.text !== undefined && src.text.trim().length > 0) nodeInputText = src.text if (src?.outputType === 'scene') nodeInputScenePath = src.filePath + if (src?.outputType === 'video') nodeInputVideoPath = src.filePath } } @@ -371,16 +376,20 @@ async function executeExtensionNode( if (isModelNode) { const isSceneInput = ext?.inputs ? ext.inputs.includes('scene') : ext?.input === 'scene' + const isVideoInput = ext?.inputs === undefined && ext?.input === 'video' const isTextInput = ext?.inputs ? ext.inputs.every((i) => i === 'text') : ext?.input === 'text' if (isSceneInput && !nodeInputScenePath) throw new Error(`${ext?.name ?? 'Model'} needs an incoming scene connection`) - const activeImagePath = (isTextInput || isSceneInput) ? undefined : (nodeInputPath ?? selectedImagePath) - if (!isTextInput && !isSceneInput && !selectedImageData && (!activeImagePath || activeImagePath.trim().length === 0)) { + if (isVideoInput && !nodeInputVideoPath) throw new Error(`${ext?.name ?? 'Model'} needs an incoming video connection`) + const activeImagePath = (isTextInput || isSceneInput || isVideoInput) ? undefined : (nodeInputPath ?? selectedImagePath) + if (!isTextInput && !isSceneInput && !isVideoInput && !selectedImageData && (!activeImagePath || activeImagePath.trim().length === 0)) { throw new Error('No input image selected for model node') } - let blob: Blob - let fname: string - if (isTextInput || isSceneInput || (selectedImageData && nodeInputPath === undefined)) { + let blob: Blob | undefined + let fname: string | undefined + if (isSceneInput || isVideoInput) { + // Typed artifacts cross the dedicated JSON boundary; never manufacture image bytes. + } else if (isTextInput || (selectedImageData && nodeInputPath === undefined)) { const base64 = selectedImageData && nodeInputPath === undefined ? selectedImageData : 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==' // 1x1 transparent PNG @@ -417,17 +426,29 @@ async function executeExtensionNode( let submission: { data: { job_id: string } } if (isSceneInput) { const normalized = nodeInputScenePath!.replace(/\\/g, '/') - const inputPath = normalized.startsWith(`${workspaceDir}/`) - ? normalized.slice(workspaceDir.length + 1) + const workspaceRoot = workspaceDir.replace(/\\/g, '/').replace(/\/+$/, '') + const inputPath = normalized.startsWith(`${workspaceRoot}/`) + ? normalized.slice(workspaceRoot.length + 1) : normalized.replace(/^\/workspace\//, '') submission = await client.post('/generate/from-artifact', { input_kind: 'scene', input_path: inputPath, model_id: node.data.extensionId ?? '', collection: 'Workflows', params: { ...effectiveParams, ...extraParams }, }) + } else if (isVideoInput) { + const normalized = nodeInputVideoPath!.replace(/\\/g, '/') + const workspaceRoot = workspaceDir.replace(/\\/g, '/').replace(/\/+$/, '') + const inputPath = normalized.startsWith(`${workspaceRoot}/`) + ? normalized.slice(workspaceRoot.length + 1) + : normalized.replace(/^\/workspace\//, '') + submission = await client.post('/generate/from-artifact', { + input_kind: 'video', input_path: inputPath, + model_id: node.data.extensionId ?? '', collection: 'Workflows', + params: { ...effectiveParams, ...extraParams }, + }) } else { const fd = new FormData() - fd.append('image', blob, fname) + fd.append('image', blob!, fname!) fd.append('model_id', node.data.extensionId ?? '') fd.append('collection', 'Workflows') fd.append('remesh', 'none') @@ -467,6 +488,7 @@ async function executeExtensionNode( if (ext?.input === 'mesh' && !nodeInputPath) throw new Error(`${ext.name} needs an incoming mesh connection`) if (ext?.input === 'image' && !nodeInputPath) throw new Error(`${ext.name} needs an incoming image connection`) if (ext?.input === 'audio' && !nodeInputPath) throw new Error(`${ext.name} needs an incoming audio connection`) + if (ext?.input === 'video' && !nodeInputPath) throw new Error(`${ext.name} needs an incoming video connection`) if (ext?.input === 'text' && !nodeInputText) throw new Error(`${ext.name} needs an incoming text connection`) const parts = (node.data.extensionId ?? '').split('/') @@ -830,6 +852,14 @@ export const useWorkflowRunStore = create((set, get) => { outputType: 'scene', }) } + if (node.type === 'videoNode') { + const workspacePath = node.data.params?.workspacePath as string | undefined + const video = normalizeVideoSource(workspacePath, workspaceDir) + if (video) nodeOutputs.set(node.id, { + filePath: video.absolutePath, + outputType: 'video', + }) + } } const ctx: RunContext = { diff --git a/src/areas/workflows/workflowVideoRun.test.mjs b/src/areas/workflows/workflowVideoRun.test.mjs new file mode 100644 index 00000000..021c8db7 --- /dev/null +++ b/src/areas/workflows/workflowVideoRun.test.mjs @@ -0,0 +1,55 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { build } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const dir = mkdtempSync(join(tmpdir(), 'modly-video-run-')) +const stub = (name, source) => { const path = join(dir, name); writeFileSync(path, source); return path } +const aliases = new Map([ + ['axios', stub('axios.ts', `const axios: any = { create: () => (globalThis as any).__client }; export default axios; export type AxiosInstance = any`)], + ['@shared/stores/appStore', stub('app.ts', `export const state: any = { apiUrl: 'x', setCurrentJob() {}, updateCurrentJob() {} }; export const useAppStore: any = (s: any) => s(state); useAppStore.getState = () => state`)], + ['./mockExtensions', stub('ext.ts', `export const getWorkflowExtension = (id: string, all: any[]) => all.find((x) => x.id === id); export type WorkflowExtension = any`)], + ['@shared/utils/notification', stub('notify.ts', `export const showCompletionNotification = async () => {}; export const showErrorNotification = async () => {}`)], +]) +const outfile = join(dir, 'store.cjs') +writeFileSync(outfile, (await build({ entryPoints: [resolve('src/areas/workflows/workflowRunStore.ts')], bundle: true, platform: 'node', format: 'cjs', write: false, plugins: [{ name: 'aliases', setup(build) { build.onResolve({ filter: /.*/ }, (args) => aliases.has(args.path) ? { path: aliases.get(args.path) } : null) } }] })).outputFiles[0].text) +const { useWorkflowRunStore } = createRequire(import.meta.url)(outfile) + +test('video model receives a typed artifact path without fake image bytes', async () => { + const posts = [] + globalThis.window = { electron: { settings: { get: async () => ({ workspaceDir: 'C:\\MODLY\\workspace' }) }, fs: { deleteDirectory: async () => ({ success: true }), listFiles: async () => [], readFileBase64: async () => { throw new Error('video must not be read as image bytes') } } } } + globalThis.__client = { post: async (url, body) => { posts.push({ url, body }); return { data: { job_id: 'video-job' } } }, get: async () => ({ data: { status: 'done', progress: 100, output_url: '/workspace/Workflows/result.glb' } }) } + const workflow = { id: 'wf', name: 'Video', description: '', createdAt: '', updatedAt: '', nodes: [ + { id: 'source', type: 'videoNode', position: { x: 0, y: 0 }, data: { enabled: true, params: { workspacePath: 'Workflows/Videos/clip.mp4' } } }, + { id: 'model', type: 'extensionNode', position: { x: 1, y: 0 }, data: { enabled: true, extensionId: 'demo/video', params: {} } }, + ], edges: [{ id: 'e', source: 'source', target: 'model' }] } + await useWorkflowRunStore.getState().run(workflow, [{ id: 'demo/video', name: 'Video', type: 'model', input: 'video', output: 'mesh', params: [] }]) + assert.deepEqual(posts[0], { url: '/generate/from-artifact', body: { input_kind: 'video', input_path: 'Workflows/Videos/clip.mp4', model_id: 'demo/video', collection: 'Workflows', params: {} } }) +}) + +test('process video input and video output keep the generic process contract', async () => { + const calls = [] + globalThis.window = { electron: { + settings: { get: async () => ({ workspaceDir: 'C:\\MODLY\\workspace' }) }, + fs: { deleteDirectory: async () => ({ success: true }), listFiles: async () => [] }, + extensions: { runProcess: async (...args) => { + calls.push(args) + return { success: true, result: { filePath: 'C:\\MODLY\\workspace\\Workflows\\result.mp4' } } + } }, + } } + globalThis.__client = { post: async () => { throw new Error('process video must not use the model API') } } + const workflow = { id: 'wf-process', name: 'Process video', description: '', createdAt: '', updatedAt: '', nodes: [ + { id: 'source', type: 'videoNode', position: { x: 0, y: 0 }, data: { enabled: true, params: { workspacePath: 'Workflows/Videos/clip.mp4' } } }, + { id: 'process', type: 'extensionNode', position: { x: 1, y: 0 }, data: { enabled: true, extensionId: 'demo/process-video', params: {} } }, + ], edges: [{ id: 'e', source: 'source', target: 'process' }] } + await useWorkflowRunStore.getState().run(workflow, [{ id: 'demo/process-video', name: 'Video process', type: 'process', input: 'video', output: 'video', params: [] }]) + assert.equal(calls.length, 1) + assert.deepEqual(calls[0], [ + 'demo', + { filePath: 'C:/MODLY/workspace/Workflows/Videos/clip.mp4', text: undefined, texts: undefined, nodeId: 'process-video' }, + {}, + ]) +}) diff --git a/src/areas/workflows/workflowVideoSource.test.mjs b/src/areas/workflows/workflowVideoSource.test.mjs new file mode 100644 index 00000000..c8f7aa90 --- /dev/null +++ b/src/areas/workflows/workflowVideoSource.test.mjs @@ -0,0 +1,20 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const outfile = join(mkdtempSync(join(tmpdir(), 'modly-video-source-')), 'video.cjs') +writeFileSync(outfile, buildSync({ entryPoints: [resolve('src/areas/workflows/workflowVideoSource.ts')], bundle: true, platform: 'node', format: 'cjs', write: false }).outputFiles[0].text) +const { normalizeVideoSource } = createRequire(import.meta.url)(outfile) + +test('Load Video accepts durable workspace video paths only', () => { + assert.deepEqual(normalizeVideoSource('Workflows/Videos/clip.mp4', '/workspace'), { + workspacePath: 'Workflows/Videos/clip.mp4', absolutePath: '/workspace/Workflows/Videos/clip.mp4', + }) + for (const value of ['../clip.mp4', '/tmp/clip.mp4', 'C:/clip.mp4', 'Workflows/%2e%2e/clip.mp4', 'Workflows/clip.exe']) { + assert.equal(normalizeVideoSource(value, '/workspace'), undefined, value) + } +}) diff --git a/src/areas/workflows/workflowVideoSource.ts b/src/areas/workflows/workflowVideoSource.ts new file mode 100644 index 00000000..8fe6f9a2 --- /dev/null +++ b/src/areas/workflows/workflowVideoSource.ts @@ -0,0 +1,24 @@ +const VIDEO_EXTENSIONS = new Set(['mp4', 'm4v', 'mov', 'webm', 'mkv', 'avi']) + +function isAbsolutePath(value: string): boolean { + return value.startsWith('/') || /^[A-Za-z]:\//.test(value) || value.startsWith('//') +} + +function isSafeRelativePath(value: string): boolean { + if (!value || value !== value.trim() || value.includes('\u0000')) return false + if (isAbsolutePath(value) || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(value) + || /%(?:25|2e|2f|5c|00)/i.test(value) || /%(?![0-9a-f]{2})/i.test(value)) return false + return value.split('/').every((part) => part.length > 0 && part !== '.' && part !== '..') +} + +export function normalizeVideoSource( + rawPath: string | undefined, + workspaceDir: string, +): { workspacePath: string; absolutePath: string } | undefined { + const workspacePath = rawPath?.replace(/\\/g, '/') + if (!workspacePath || !isSafeRelativePath(workspacePath)) return undefined + const extension = workspacePath.split('.').pop()?.toLowerCase() + if (!extension || !VIDEO_EXTENSIONS.has(extension)) return undefined + const root = workspaceDir.replace(/\\/g, '/').replace(/\/+$/, '') + return { workspacePath, absolutePath: `${root}/${workspacePath}` } +} diff --git a/src/shared/stores/workflowsStore.ts b/src/shared/stores/workflowsStore.ts index f6fb829c..0579f3e3 100644 --- a/src/shared/stores/workflowsStore.ts +++ b/src/shared/stores/workflowsStore.ts @@ -98,7 +98,7 @@ interface LegacyWorkflow { // Source-only nodes have no target handle; sink-only nodes have no source handle. // An edge into/out of the wrong side can't resolve a handle and makes React Flow // warn ("Couldn't create edge for target handle id: null") on every render. -export const NODE_TYPES_WITHOUT_TARGET = new Set(['imageNode', 'textNode', 'meshNode', 'sceneNode', 'inputNode', 'forEachNode']) +export const NODE_TYPES_WITHOUT_TARGET = new Set(['imageNode', 'textNode', 'meshNode', 'sceneNode', 'videoNode', 'inputNode', 'forEachNode']) export const NODE_TYPES_WITHOUT_SOURCE = new Set(['outputNode', 'previewNode']) function sanitizeEdges(nodes: WFNode[], edges: WFEdge[]): WFEdge[] { diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index c171b664..c38e5fd7 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -14,10 +14,10 @@ import type { export interface ExtensionNode { id: string name: string - input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' - inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene')[] // multi-input nodes; overrides input when set + input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' + inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video')[] // multi-input nodes; overrides input when set inputLabels?: string[] // display labels per input slot (e.g. positive/negative) - output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' + output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' paramsSchema: ParamSchema[] paramDefaults?: Record hfRepo?: string @@ -200,6 +200,7 @@ declare global { fs: { getPathForFile: (file: File) => string selectImage: () => Promise + selectVideo: () => Promise<{ workspacePath: string; absolutePath: string } | null> selectMeshFile: () => Promise saveModel: (defaultName: string) => Promise readFileBase64: (filePath: string) => Promise