Sync JavaScript packages alongside Python and Rust members

README.md

@@ -119,6 +119,8 @@ Each sync also regenerates the `[patch]` entries in the workspace's `.cargo/conf
 
 When `sccache` is installed, sync also configures it as Cargo's `rustc-wrapper`, allowing unchanged compilation units to be reused across the workspace's otherwise-independent Cargo target directories. An existing wrapper is always preserved.
 
+JavaScript packages join the same sync. A `package.json` in a workspace repo dir makes that dir a JS member (`felt/package.json`); an immediate subdirectory of a repo dir with a `package.json` is also a member (`mdhtml/wasm`), unless it carries its own lockfile (`bun.lock`, `package-lock.json`, `yarn.lock`, or `pnpm-lock.yaml`: a package that manages itself) or its root-relative path matches `[tool.fastws].exclude` (`ghapi/examples`). A repo dir's own lockfile never opts it out: every checkout is in the workspace. `node_modules`, `pkg`, and `_`-prefixed dirs are skipped. Sync prints the members it adds, so a stray package is visible the first time. Each sync regenerates the `workspaces` list in the root `package.json`, creating the file the first time a member exists and keeping entries that point outside the root or use globs. After `uv sync` it runs the package manager's `install` at the root, so every member resolves its siblings through the root `node_modules` symlinks: the npm analog of editable installs. A member with a `Cargo.toml` beside its `package.json` is a native package (a wasm build). Sync runs its `build` script when `pkg/` is missing or older than any source in the member's repo, the parent crate included, which is the JS analog of `maturin develop`. The package manager is `npm` unless `[tool.fastws]` in the workspace `pyproject.toml` sets `js = "bun"`; both read the same `workspaces` field, and sync stops with a message when the chosen tool is not installed. A checkout that is only a JS package is a valid member: it is excluded from the uv workspace like a Cargo-only crate and is never treated as a pending scaffold. A Vite app in the workspace needs `server.fs.allow` set to the workspace root (`searchForWorkspaceRoot(process.cwd())` from `vite`) when a linked package fetches a file at runtime, such as a `.wasm`, because the symlink resolves to the sibling checkout and SvelteKit's default allow list stops at the app and its `node_modules`.
+
 At most once per day (tracked by a stamp file inside the workspace's `.git`, so git never sees it), the sync also floats dependencies: `uv sync -U` instead of plain `uv sync`, plus a parallel `cargo update` in every member with a `Cargo.toml`, printing what moved. Pass `--upgrade` to force that pass regardless of when it last ran.
 
 Before every uv sync, `ws-sync` writes `.git/fastws-cargo-key` for each member crate. The key hashes `Cargo.lock` contents and the workspace Cargo patch configuration. For Git dependencies redirected to local paths by `[patch."<url>"]`, it also hashes each patched crate's `Cargo.toml`, `build.rs`, and `src` tree, recursively. The file is rewritten only when that content changes, so projects can use `{ file = ".git/fastws-cargo-key" }` in `tool.uv.cache-keys` without rebuilding after a timestamp-only `Cargo.lock` write.

fastws/core.py

@@ -84,6 +84,12 @@ def _ws_cfg(root: Path):
     exclude = ws.get("exclude") or []
     return members, exclude
 
+def _fastws_cfg(root: Path) -> dict:
+    "The `[tool.fastws]` table from the workspace root pyproject.toml (empty when absent)."
+    pyproj = root/"pyproject.toml"
+    if not pyproj.exists(): return {}
+    return tomllib.loads(pyproj.read_text(encoding="utf-8")).get("tool", {}).get("fastws", {})
+
 def _matches_ws(name: str, pattern: str) -> bool:
     pattern = pattern.strip()
     return any(fnmatch.fnmatch(candidate, normalized) for candidate in (name, f"./{name}") for normalized in (pattern, pattern.removeprefix("./")))
@@ -176,13 +182,21 @@ def _cargo_only(d: Path) -> bool:
     "A Rust crate with no Python layer: not a pending scaffold, just not a uv workspace member"
     return (d/"Cargo.toml").exists() and not (d/"pyproject.toml").exists()
 
+def _npm_only(d: Path) -> bool:
+    "A JS package with no Python layer: not a pending scaffold, just not a uv workspace member"
+    return (d/"package.json").exists() and not (d/"pyproject.toml").exists()
+
+def _pending_dirs(root: Path) -> list[str]:
+    "uv workspace dirs that are not Python projects yet; sync stops rather than let uv fail on them"
+    return [d.name for d in _ws_dirs(root) if not (d/"pyproject.toml").exists()]
+
 def _sync_ws_excludes(pyproject_path: Path, root: Path, tracked: set[str]) -> tuple[list[str], list[str]]:
     """Regenerate `tool.uv.workspace.exclude` and return (added, removed).
 
     Kept as-is: `[tool.fastws].exclude` entries (intent), globs, missing dirs, and tracked dirs
     (repos.txt checkouts) that are still not valid Python projects. Auto-managed: entries for other
     existing dirs are regenerated each sync, excluding dirs without a valid pyproject (tracked dirs
-    only when they are Cargo-only crates, since a tracked dir with neither file is a pending member
+    only when they are Cargo-only crates or npm-only packages, since a tracked dir with none of those files is a pending member
     awaiting scaffolding) and un-excluding dirs that gained one; deliberately excluding a real
     project takes a `[tool.fastws]` entry."""
     if not pyproject_path.exists(): return [], []
@@ -197,7 +211,7 @@ def _sync_ws_excludes(pyproject_path: Path, root: Path, tracked: set[str]) -> tu
     auto = [d.name for d in sorted(root.iterdir())
             if d.is_dir() and not d.name.startswith(".") and any(_matches_ws(d.name, m) for m in members)
             and not any(_matches_ws(d.name, e) for e in kept)
-            and (d.name not in tracked or _cargo_only(d)) and not _valid_project_dir(d)]
+            and (d.name not in tracked or _cargo_only(d) or _npm_only(d)) and not _valid_project_dir(d)]
     survivors = set(kept) | set(auto)
     new = [e for e in cur if e in survivors] + [e for e in kept + auto if e not in cur]
     if new == cur: return [], []
@@ -437,11 +451,11 @@ def ws_branches(
 
 _BUILD_SKIP_DIRS = {".git", "__pycache__", ".venv", "node_modules", "dist", "build", "target", ".ipynb_checkpoints", ".pytest_cache", ".mypy_cache"}
 
-def _src_mtime(d: Path) -> float:
-    "Newest file mtime under `d`, ignoring VCS internals and build outputs"
+def _src_mtime(d: Path, skip: set[str] = _BUILD_SKIP_DIRS) -> float:
+    "Newest file mtime under `d`, ignoring VCS internals and build outputs (the dir names in `skip`)"
     newest = 0.0
     for dirpath, dirnames, filenames in os.walk(d):
-        dirnames[:] = [o for o in dirnames if o not in _BUILD_SKIP_DIRS and not o.endswith(".egg-info")]
+        dirnames[:] = [o for o in dirnames if o not in skip and not o.endswith(".egg-info")]
         for f in filenames:
             try: newest = max(newest, os.stat(os.path.join(dirpath, f)).st_mtime)
             except OSError: pass
@@ -689,6 +703,61 @@ def upd(d):
             elif lines := [l for l in out.splitlines() if l.lstrip().startswith(("Updating ", "Adding ", "Removing ")) and "crates.io index" not in l]:
                 print(f"{name}:\n" + "\n".join(lines))
 
+_JS_SKIP = {"node_modules", "pkg"}
+_JS_LOCKS = ("bun.lock", "package-lock.json", "yarn.lock", "pnpm-lock.yaml")
+
+def _npm_dirs(root: Path) -> list[Path]:
+    """JS members: a root repo dir with a package.json, else its immediate subdirs with a package.json.
+
+    A subdir with its own lockfile manages itself and stays out; a root repo is always in. `[tool.fastws].exclude`
+    matches the root-relative path (`ghapi/examples`). `_`-prefixed names and build outputs are skipped."""
+    exclude = _fastws_cfg(root).get("exclude", [])
+    def ok(d): return d.is_dir() and not d.name.startswith((".", "_")) and d.name not in _JS_SKIP
+    def excluded(d): return any(_matches_ws(os.path.relpath(d, root), e) for e in exclude)
+    res = []
+    for d in sorted(root.iterdir()):
+        if not ok(d) or excluded(d): continue
+        if (d/"package.json").exists(): res.append(d)
+        else: res += [p for p in sorted(d.iterdir()) if ok(p) and (p/"package.json").exists() and not excluded(p) and not any((p/l).exists() for l in _JS_LOCKS)]
+    return res
+
+def _sync_ws_package_json(root: Path, members: list[Path]) -> tuple[list[str], list[str]]:
+    """Regenerate `workspaces` in the root package.json (created when first needed) and return (added, removed).
+
+    Entries pointing outside `root` and globs are kept as-is. Entries for dirs inside it are regenerated
+    from `members`, so the JS install links every discovered package: the npm analog of editable installs.
+    Only the list form of `workspaces` is managed; the object form (`{"packages": [...]}`) is not supported."""
+    path = root/"package.json"
+    data = json.loads(path.read_text()) if path.exists() else {"private": True}
+    cur = list(data.get("workspaces") or [])
+    kept = [e for e in cur if any(c in e for c in "*?[") or not (root/e).resolve().is_relative_to(root.resolve())]
+    new = kept + [e for e in (os.path.relpath(d, root) for d in members) if e not in kept]
+    if new == cur: return [], []
+    data["workspaces"] = new
+    path.write_text(json.dumps(data, indent=2) + "\n")
+    return [e for e in new if e not in cur], [e for e in cur if e not in new]
+
+def _js_tool(root: Path) -> str:
+    "The JS package manager: `[tool.fastws].js` in the workspace pyproject, default npm; bun reads the same `workspaces` list"
+    tool = _fastws_cfg(root).get("js", "npm")
+    if tool not in ("npm", "bun"): raise SystemExit(f"[tool.fastws].js must be npm or bun, not {tool!r}")
+    return tool
+
+def _js_stale(root: Path, d: Path) -> bool:
+    "Does JS member `d` need its build script? Only members with a Cargo.toml build natively, into `pkg/`, which is stale when missing or older than any source in the member's repo (the parent crate included)"
+    if not (d/"Cargo.toml").exists(): return False
+    pkg, repo = d/"pkg", d if d.parent.resolve() == root.resolve() else d.parent
+    return not pkg.exists() or _src_mtime(pkg) < _src_mtime(repo, _BUILD_SKIP_DIRS | _JS_SKIP)
+
+def _sync_js(root: Path, members: list[Path]) -> list[Path]:
+    "Install the JS workspace at `root`, then run the build script of each native member whose output is stale (the JS analog of `maturin develop`); returns the members built"
+    tool = _js_tool(root)
+    if not shutil.which(tool): raise SystemExit(f"{tool} is not installed: install it, or set [tool.fastws].js to a package manager that is")
+    subprocess.run([tool, "install"], check=True, cwd=root)
+    built = [d for d in members if _js_stale(root, d)]
+    for d in built: subprocess.run([tool, "run", "build"], check=True, cwd=d)
+    return built
+
 @call_parse
 async def ws_sync(
     workspace: str = "",  # Workspace root; defaults to active venv parent when available
@@ -725,14 +794,20 @@ async def ws_sync(
     if removed_p: print(f"Cargo patches removed: {', '.join(removed_p)}")
     if wrapper_added: print("Cargo builds now use sccache")
 
-    if bad := [d.name for d in _ws_dirs(root) if not (d/"pyproject.toml").exists()]:
+    js_members = _npm_dirs(root)
+    added_j, removed_j = _sync_ws_package_json(root, js_members)
+    if added_j: print(f"JS workspace packages added: {', '.join(added_j)}")
+    if removed_j: print(f"JS workspace packages removed: {', '.join(removed_j)}")
+
+    if bad := _pending_dirs(root):
         print(f"⚠️  Skipping uv sync, not Python projects yet (scaffold with e.g. nbdev-new or ship-new, or remove): {', '.join(bad)}")
         return
     up = upgrade or _should_upgrade(root)
     if up: _cargo_update(root, workers=workers)
     _sync_cargo_keys(root)
     subprocess.run(["uv", "sync", "-U"] if up else ["uv", "sync"], check=True, cwd=root)
     if up: _upgrade_stamp(root).touch()
+    if js_members and (built := _sync_js(root, js_members)): print(f"JS packages built: {', '.join(os.path.relpath(d, root) for d in built)}")
 
 @call_parse
 async def ws_add(

fastws/releases.py

@@ -10,21 +10,12 @@
 from ghapi.core import APIError, GhApi, dep_closure, local_dep_graph
 from packaging.version import InvalidVersion, Version
 
-from .core import _load_repo_entries, _load_repos, _resolve_path, _ws_root
-
-try: import tomllib
-except ModuleNotFoundError: import tomli as tomllib
+from .core import _fastws_cfg, _load_repo_entries, _load_repos, _resolve_path, _ws_root
 
 # Start-of-message regexes for commits that need no release (curation noise, CI config, docs regen)
 DEFAULT_SKIP = ["bump$", "Bump version to ", "meta", "auto", "README", "regen readme", "nbdev regen",
     "update .gitignore", r"\.?gitignore$", "workflow", "chkstyle skip", "tests$", "md$", "docs?$", "clean$", "CI$", "ignore$", "allowed_metadata_keys$"]
 
-def _fastws_cfg(root: Path) -> dict:
-    "The `[tool.fastws]` table from the workspace root pyproject.toml (empty when absent)."
-    pyproj = root/"pyproject.toml"
-    if not pyproj.exists(): return {}
-    return tomllib.loads(pyproj.read_text(encoding="utf-8")).get("tool", {}).get("fastws", {})
-
 def _skip_pats(skip=None, root: Path|None = None) -> list[re.Pattern]:
     "Compiled skip patterns: defaults + `[tool.fastws].release_skip` from the workspace pyproject + `skip`."
     pats = list(DEFAULT_SKIP)

tests/test_sync.py

@@ -1,4 +1,4 @@
-import os, pytest
+import json, os, pytest, time
 from types import SimpleNamespace
 from fastgit import Git
 import fastws.core as core
@@ -376,3 +376,115 @@ def test_sync_cargo_wrapper(tmp_path, monkeypatch):
     config.write_text('[build]\nrustc-wrapper = "other-cache"\n')
     assert not core._sync_cargo_wrapper(tmp_path)
     assert core.tomllib.loads(config.read_text())['build']['rustc-wrapper'] == 'other-cache'
+
+
+def test_npm_dirs_discovers_root_and_subdir_packages(tmp_path):
+    for name in ('app', 'crate', 'plain', '_scratch', 'node_modules'): (tmp_path/name).mkdir()
+    (tmp_path/'app'/'package.json').write_text('{}')
+    (tmp_path/'app'/'tools').mkdir()
+    (tmp_path/'app'/'tools'/'package.json').write_text('{}')  # a root package owns its subdirs: not a separate member
+    (tmp_path/'app'/'package-lock.json').write_text('{}')  # a repo's own lockfile does not opt it out: every checkout is in the workspace
+    for name in ('wasm', 'node_modules', 'pkg', '_tmp', 'docs', 'frontend'): (tmp_path/'crate'/name).mkdir()
+    for name in ('wasm', 'node_modules', 'pkg', '_tmp', 'docs', 'frontend'): (tmp_path/'crate'/name/'package.json').write_text('{}')
+    (tmp_path/'crate'/'wasm'/'Cargo.toml').write_text('')  # native binding; docs is a plain package and joins the same way
+    (tmp_path/'crate'/'frontend'/'bun.lock').write_text('')  # a subdir with its own lockfile manages itself
+    (tmp_path/'_scratch'/'package.json').write_text('{}')
+    (tmp_path/'node_modules'/'package.json').write_text('{}')
+
+    assert core._npm_dirs(tmp_path) == [tmp_path/'app', tmp_path/'crate'/'docs', tmp_path/'crate'/'wasm']
+
+
+def test_npm_dirs_honours_fastws_exclude(tmp_path):
+    (tmp_path/'pyproject.toml').write_text('[tool.fastws]\nexclude = ["app", "tool*", "py/examples"]\n')
+    for name in ('app', 'lib', 'tools'):
+        (tmp_path/name).mkdir()
+        (tmp_path/name/'package.json').write_text('{}')
+    for name in ('examples', 'wasm'):
+        (tmp_path/'py'/name).mkdir(parents=True)
+        (tmp_path/'py'/name/'package.json').write_text('{}')  # examples is excluded by its root-relative path
+
+    assert core._npm_dirs(tmp_path) == [tmp_path/'lib', tmp_path/'py'/'wasm']
+
+
+def test_js_tool_default_npm_rejects_others(tmp_path):
+    assert core._js_tool(tmp_path) == 'npm'
+    (tmp_path/'pyproject.toml').write_text('[tool.fastws]\njs = "bun"\n')
+    assert core._js_tool(tmp_path) == 'bun'
+    (tmp_path/'pyproject.toml').write_text('[tool.fastws]\njs = "pnpm"\n')
+    with pytest.raises(SystemExit, match='pnpm'): core._js_tool(tmp_path)
+
+
+def test_sync_ws_package_json_generates_and_preserves(tmp_path):
+    pkg = tmp_path/'package.json'
+    pkg.write_text('{\n  "name": "ws",\n  "private": true,\n  "workspaces": ["../outside", "gone", "tools/*"]\n}\n')
+    members = [tmp_path/'app', tmp_path/'crate'/'wasm']
+
+    added, removed = core._sync_ws_package_json(tmp_path, members)
+    data = json.loads(pkg.read_text())
+    # kept: entries outside the root and globs; managed: existing dirs regenerated from discovery
+    assert data['workspaces'] == ['../outside', 'tools/*', 'app', 'crate/wasm']
+    assert data['name'] == 'ws' and data['private'] is True  # other keys untouched
+    assert added == ['app', 'crate/wasm'] and removed == ['gone']
+
+    content = pkg.read_text()
+    assert core._sync_ws_package_json(tmp_path, members) == ([], [])
+    assert pkg.read_text() == content
+
+    # a missing root package.json is created
+    bare = tmp_path/'ws2'
+    bare.mkdir()
+    assert core._sync_ws_package_json(bare, [bare/'a']) == (['a'], [])
+    assert json.loads((bare/'package.json').read_text()) == {'private': True, 'workspaces': ['a']}
+
+
+def test_ws_excludes_treat_npm_only_dirs_like_cargo_only(tmp_path):
+    pyproject = tmp_path/'pyproject.toml'
+    pyproject.write_text('[project]\nname = "uvws"\n\n[tool.uv.workspace]\nmembers = ["./*"]\nexclude = []\n')
+    for name in ('app', 'pending'): (tmp_path/name).mkdir()
+    (tmp_path/'app'/'package.json').write_text('{}')
+
+    # a tracked npm-only checkout is a valid JS project: excluded from uv, never pending; an empty tracked dir still awaits scaffolding
+    assert core._sync_ws_excludes(pyproject, tmp_path, {'app', 'pending'}) == (['app'], [])
+    assert core._pending_dirs(tmp_path) == ['pending']
+
+
+def test_sync_js_installs_then_builds_stale_native_members(tmp_path, monkeypatch):
+    (tmp_path/'pyproject.toml').write_text('[tool.fastws]\njs = "bun"\n')
+    monkeypatch.setattr(core.shutil, 'which', lambda t: f'/usr/bin/{t}')
+    app = tmp_path/'app'
+    app.mkdir()
+    (app/'package.json').write_text('{}')
+    crate = tmp_path/'crate'
+    wasm = crate/'wasm'
+    for d in (crate/'src', wasm/'src'): d.mkdir(parents=True)
+    (crate/'Cargo.toml').write_text('[package]\nname = "crate"\n')
+    (crate/'src'/'lib.rs').write_text('')
+    (wasm/'Cargo.toml').write_text('[package]\nname = "crate-wasm"\n')
+    (wasm/'package.json').write_text('{}')
+    (wasm/'src'/'lib.rs').write_text('')
+    calls = []
+    monkeypatch.setattr(core.subprocess, 'run', lambda cmd, **kw: calls.append((cmd, kw.get('cwd'))))
+    members = [app, wasm]
+
+    # a tool that is not installed stops the sync with a one-line message before anything runs
+    monkeypatch.setattr(core.shutil, 'which', lambda t: None)
+    with pytest.raises(SystemExit, match='bun'): core._sync_js(tmp_path, members)
+    assert calls == []
+    monkeypatch.setattr(core.shutil, 'which', lambda t: f'/usr/bin/{t}')
+
+    # install at the root with the configured tool; a native member with no pkg/ yet is built
+    assert core._sync_js(tmp_path, members) == [wasm]
+    assert calls == [(['bun', 'install'], tmp_path), (['bun', 'run', 'build'], wasm)]
+
+    # a pkg/ newer than every source in the member's repo means nothing to rebuild
+    (wasm/'pkg').mkdir()
+    (wasm/'pkg'/'out.wasm').write_text('')
+    now = time.time()
+    os.utime(wasm/'pkg'/'out.wasm', (now+10, now+10))
+    calls.clear()
+    assert core._sync_js(tmp_path, members) == []
+    assert calls == [(['bun', 'install'], tmp_path)]
+
+    # a change anywhere in that repo, including the parent crate the wasm depends on, makes it stale again
+    os.utime(crate/'src'/'lib.rs', (now+20, now+20))
+    assert core._sync_js(tmp_path, members) == [wasm]