diff --git a/README.md b/README.md index 4161394..552dee0 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ By default, `ws-sync` uses the active venv's parent as the workspace root. It cr Project scanning respects `tool.uv.workspace.members` and `exclude`. If a member directory lacks `pyproject.toml`, such as a fresh empty clone, sync warns and skips the `uv sync` step. -The workspace `exclude` list is maintained automatically. An unlisted top-level directory that is not a valid Python project is excluded until it has a `pyproject.toml`. A `repos.txt` checkout is automatically excluded only when it is a Cargo-only Rust crate. A listed checkout with neither `pyproject.toml` nor `Cargo.toml` is treated as a pending member and triggers the warning above. +Fastws maintains `tool.uv.workspace.exclude`. It excludes unlisted top-level directories without a valid Python project. It also excludes listed Rust or JavaScript projects without `pyproject.toml`. For a listed checkout with none of `pyproject.toml`, `Cargo.toml`, or `package.json`, sync warns and skips installation. Existing globs, entries for missing directories, and entries for checkouts that are still not Python projects are retained. Use `exclude = [...]` under `[tool.fastws]` to specify exclusions the scan cannot infer, such as keeping a valid project out of the workspace. Adding members preserves hand-written `[tool.uv.sources]` entries, including path and Git sources. @@ -179,6 +179,14 @@ Do not commit a `Cargo.lock` generated under these patches. Its source-less loca When `sccache` is installed, sync also configures it as Cargo's `rustc-wrapper`, allowing unchanged compilation units to be reused across the workspace's otherwise-independent Cargo target directories. An existing wrapper is always preserved. +Fastws discovers JavaScript packages directly under the workspace root. It reads each package's `workspaces` list for declared nested packages. These lists accept paths and globs. Fastws ignores undeclared nested packages. Lockfiles do not affect discovery. `[tool.fastws].exclude` matches paths relative to the workspace root. Fastws skips `node_modules` and directories whose names start with `.` or `_`. + +Each sync updates `workspaces` in the root `package.json`. It creates the file when needed. It preserves external paths, globs, and unrelated settings. It reports added and removed packages. + +After `uv sync`, fastws runs ` install` at the workspace root. It then runs ` run build` in each JavaScript package containing both `Cargo.toml` and a `build` script. Cargo handles incremental compilation. Other build steps still run. + +The package manager defaults to `npm`. Set `js` under `[tool.fastws]` in the workspace `pyproject.toml` to choose another executable, such as `js = "bun"`. Fastws requires a list for `workspaces` in `package.json`, following the npm format. It stops if it cannot find the selected executable. + At most once per day, sync upgrades dependencies with `uv sync -U` and a parallel `cargo update` in every member with a `Cargo.toml`. It prints the changes and records the run in a stamp file inside the workspace's `.git` directory. Pass `--upgrade` to force an upgrade regardless of the last run time. Before every uv sync, `ws-sync` writes `.git/fastws-cargo-key` for each member crate. The key hashes `Cargo.lock` contents and the workspace Cargo patch configuration. For Git dependencies redirected to local paths by `[patch.""]`, it also hashes each patched crate's `Cargo.toml`, `build.rs`, and `src` tree, recursively. The file is rewritten only when that content changes, so projects can use `{ file = ".git/fastws-cargo-key" }` in `tool.uv.cache-keys` without rebuilding after a timestamp-only `Cargo.lock` write. diff --git a/fastws/core.py b/fastws/core.py index 16676fa..00b7715 100644 --- a/fastws/core.py +++ b/fastws/core.py @@ -2,7 +2,7 @@ __all__ = ["ws_setup", "ws_clone", "ws_pull", "ws_status", "ws_branches", "ws_build", "ws_sync", "ws_add", "ws_remove"] -import ast, fnmatch, hashlib, json, os, re, shlex, shutil, subprocess, sys, time +import ast, fnmatch, glob, hashlib, json, os, re, shlex, shutil, subprocess, sys, time from pathlib import Path from concurrent.futures import ThreadPoolExecutor @@ -98,16 +98,25 @@ def _ws_cfg(root: Path): exclude = ws.get("exclude") or [] return members, exclude +def _fastws_cfg(root: Path) -> dict: + "The `[tool.fastws]` table from the workspace root pyproject.toml (empty when absent)." + pyproj = root/"pyproject.toml" + if not pyproj.exists(): return {} + return tomllib.loads(pyproj.read_text(encoding="utf-8")).get("tool", {}).get("fastws", {}) + def _matches_ws(name: str, pattern: str) -> bool: pattern = pattern.strip() return any(fnmatch.fnmatch(candidate, normalized) for candidate in (name, f"./{name}") for normalized in (pattern, pattern.removeprefix("./"))) -def _is_ws_dir(d: Path, members, exclude) -> bool: - return d.is_dir() and not d.name.startswith(".") and any(_matches_ws(d.name, o) for o in members) and not any(_matches_ws(d.name, o) for o in exclude) +def _project_dirs(root: Path, manifest="", members=("*",), exclude=()) -> list[Path]: + "Expand member patterns into unique directories, optionally requiring a manifest; exclusions are relative to root." + dirs = dict.fromkeys(d for pat in members for d in sorted(root.glob(pat))) + return [d for d in dirs if d.is_dir() and (not manifest or (d/manifest).is_file()) + and not any(_matches_ws(os.path.relpath(d, root), e) for e in exclude)] def _ws_dirs(root: Path) -> list[Path]: members, exclude = _ws_cfg(root) - return [d for d in sorted(root.iterdir()) if _is_ws_dir(d, members, exclude)] + return [d for d in _project_dirs(root, exclude=exclude) if not d.name.startswith(".") and any(_matches_ws(d.name, o) for o in members)] def _root_git_dirs(root: Path) -> list[Path]: "Every git checkout directly under `root`, whatever the uv workspace config; `_`-prefixed dirs are private" @@ -178,16 +187,20 @@ def _external_projects(root: Path, dirs: list[Path]) -> list[tuple[str,str]]: res = [] for d in dirs: if not d.is_dir(): continue - cands = [d] if (d/"pyproject.toml").exists() else sorted(p for p in d.iterdir() if p.is_dir() and not p.name.startswith((".","_")) and (p/"pyproject.toml").exists()) + cands = [d] if (d/"pyproject.toml").exists() else [p for p in _project_dirs(d, "pyproject.toml") if not p.name.startswith((".", "_"))] for c in cands: if name := _read_pyproject_name(c/"pyproject.toml"): res.append((name, os.path.relpath(c, root))) return res def _valid_project_dir(d: Path) -> bool: return (d/"pyproject.toml").exists() and bool(_read_pyproject_name(d/"pyproject.toml")) -def _cargo_only(d: Path) -> bool: - "A Rust crate with no Python layer: not a pending scaffold, just not a uv workspace member" - return (d/"Cargo.toml").exists() and not (d/"pyproject.toml").exists() +def _non_python_project(d: Path) -> bool: + "A Rust or JS project without a Python layer is not a pending Python scaffold." + return not (d/"pyproject.toml").exists() and any((d/f).exists() for f in ("Cargo.toml", "package.json")) + +def _pending_dirs(root: Path) -> list[str]: + "uv workspace dirs that are not Python projects yet; sync stops rather than let uv fail on them" + return [d.name for d in _ws_dirs(root) if not (d/"pyproject.toml").exists()] def _sync_ws_excludes(pyproject_path: Path, root: Path, tracked: set[str]) -> tuple[list[str], list[str]]: """Regenerate `tool.uv.workspace.exclude` and return (added, removed). @@ -195,7 +208,7 @@ def _sync_ws_excludes(pyproject_path: Path, root: Path, tracked: set[str]) -> tu Kept as-is: `[tool.fastws].exclude` entries (intent), globs, missing dirs, and tracked dirs (repos.txt checkouts) that are still not valid Python projects. Auto-managed: entries for other existing dirs are regenerated each sync, excluding dirs without a valid pyproject (tracked dirs - only when they are Cargo-only crates, since a tracked dir with neither file is a pending member + only when they are Cargo-only crates or npm-only packages, since a tracked dir with none of those files is a pending member awaiting scaffolding) and un-excluding dirs that gained one; deliberately excluding a real project takes a `[tool.fastws]` entry.""" if not pyproject_path.exists(): return [], [] @@ -207,10 +220,8 @@ def _sync_ws_excludes(pyproject_path: Path, root: Path, tracked: set[str]) -> tu intent = [e for e in data.get("tool", {}).get("fastws", {}).get("exclude", []) if isinstance(e, str)] kept = [e for e in cur if e in intent or any(c in e for c in "*?[") or not (root/e).is_dir() or (e in tracked and not _valid_project_dir(root/e))] kept += [e for e in intent if e not in kept] - auto = [d.name for d in sorted(root.iterdir()) # chkstyle: ignore-node - if d.is_dir() and not d.name.startswith(".") and any(_matches_ws(d.name, m) for m in members) - and not any(_matches_ws(d.name, e) for e in kept) - and (d.name not in tracked or _cargo_only(d)) and not _valid_project_dir(d)] + auto = [d.name for d in _project_dirs(root, exclude=kept) if not d.name.startswith(".") and any(_matches_ws(d.name, m) for m in members) + and (d.name not in tracked or _non_python_project(d)) and not _valid_project_dir(d)] survivors = set(kept) | set(auto) new = [e for e in cur if e in survivors] + [e for e in kept + auto if e not in cur] if new == cur: return [], [] @@ -569,7 +580,7 @@ def _should_upgrade(root: Path, max_age: float = 86400) -> bool: def _crate_dirs(root: Path) -> list[Path]: "Root dirs containing a Cargo.toml: the crate view of the workspace, independent of uv membership" - return [d for d in sorted(root.iterdir()) if d.is_dir() and not d.name.startswith((".", "_")) and (d/"Cargo.toml").exists()] + return [d for d in _project_dirs(root, "Cargo.toml") if not d.name.startswith((".", "_"))] def _cargo_patches(root: Path): "Local Cargo patches keyed by normalized Git URL and package name, plus their config file." @@ -589,12 +600,10 @@ def _crate_pkgs(d: Path): try: data = tomllib.loads((d/"Cargo.toml").read_text()) except tomllib.TOMLDecodeError: return if name := data.get("package", {}).get("name"): yield name, d - for pat in data.get("workspace", {}).get("members", []): - for m in sorted(d.glob(pat)): - if not (m/"Cargo.toml").exists(): continue - try: sub = tomllib.loads((m/"Cargo.toml").read_text()) - except tomllib.TOMLDecodeError: continue - if name := sub.get("package", {}).get("name"): yield name, m + for m in _project_dirs(d, "Cargo.toml", data.get("workspace", {}).get("members", [])): + try: sub = tomllib.loads((m/"Cargo.toml").read_text()) + except tomllib.TOMLDecodeError: continue + if name := sub.get("package", {}).get("name"): yield name, m def _local_crates(root: Path) -> dict[str, Path]: "Package name -> dir for every crate under `root`, nested cargo workspace members included" @@ -623,6 +632,10 @@ def _strip_patch_tables(content: str) -> str: content = content[:m.start()] + content[end:] return content +def _entry_changes(current, new): + "Added and removed keys, in the order supplied." + return [k for k in new if k not in current], [k for k in current if k not in new] + def _sync_cargo_patches(root: Path) -> tuple[list[str], list[str]]: """Regenerate `[patch]` entries in the workspace `.cargo/config.toml` and return (added, removed). @@ -659,7 +672,8 @@ def inside(spec): config.write_text(new) before = {n for entries in old.values() for n in entries} after = {n for entries in tables.values() for n in entries} - return sorted(after - before), sorted(before - after) + added, removed = _entry_changes(before, after) + return sorted(added), sorted(removed) def _sync_cargo_wrapper(root: Path) -> bool: "Add sccache to the generated Cargo config when installed, without overriding another wrapper" @@ -744,6 +758,41 @@ def upd(d): elif lines := [l for l in out.splitlines() if l.lstrip().startswith(("Updating ", "Adding ", "Removing ")) and "crates.io index" not in l]: print(f"{name}:\n" + "\n".join(lines)) +def _npm_dirs(root: Path) -> list[Path]: + "Find root JavaScript packages and their declared workspace members." + exclude = _fastws_cfg(root).get("exclude", []) + def ok(d): return not d.name.startswith((".", "_")) and d.name != "node_modules" + res = [] + for d in _project_dirs(root, "package.json", exclude=exclude): + if not ok(d): continue + data = json.loads((d/"package.json").read_text()) + members = [glob.escape(d.name)] + [f"{glob.escape(d.name)}/{p}" for p in data.get("workspaces", [])] + res += _project_dirs(root, "package.json", members, exclude) + return list(dict.fromkeys(d for d in res if ok(d))) + +def _sync_ws_package_json(root: Path, members: list[Path]) -> tuple[list[str], list[str]]: + "Update package.json workspaces, preserving external paths and globs; return (added, removed)." + path = root/"package.json" + data = json.loads(path.read_text()) if path.exists() else {"private": True} + cur = data.get("workspaces", []) + if not isinstance(cur, list): raise SystemExit(f"{path}: fastws requires `workspaces` to be a list of package paths.") + kept = [e for e in cur if any(c in e for c in "*?[") or not (root/e).resolve().is_relative_to(root.resolve())] + new = kept + [e for e in (os.path.relpath(d, root) for d in members) if e not in kept] + if new == cur: return [], [] + data["workspaces"] = new + path.write_text(json.dumps(data, indent=2) + "\n") + return _entry_changes(cur, new) + +def _sync_js(root: Path, members: list[Path]) -> list[Path]: + "Install the JS workspace, run every native member's build script, and return those members. Cargo handles incremental compilation." + tool = _fastws_cfg(root).get("js", "npm") + if not shutil.which(tool): raise SystemExit(f"{tool} is not installed: install it, or set [tool.fastws].js to a package manager that is") + subprocess.run([tool, "install"], check=True, cwd=root) + built = [d for d in members if (d/"Cargo.toml").exists() + and json.loads((d/"package.json").read_text()).get("scripts", {}).get("build")] + for d in built: subprocess.run([tool, "run", "build"], check=True, cwd=d) + return built + @call_parse async def ws_sync( workspace: str = "", # Workspace root; defaults to active venv parent when available @@ -785,7 +834,12 @@ async def ws_sync( if removed_p: print(f"Cargo patches removed: {', '.join(removed_p)}") if wrapper_added: print("Cargo builds now use sccache") - if bad := [d.name for d in _ws_dirs(root) if not (d/"pyproject.toml").exists()]: + js_members = _npm_dirs(root) + added_j, removed_j = _sync_ws_package_json(root, js_members) + if added_j: print(f"JS workspace packages added: {', '.join(added_j)}") + if removed_j: print(f"JS workspace packages removed: {', '.join(removed_j)}") + + if bad := _pending_dirs(root): print(f"⚠️ Skipping uv sync, not Python projects yet (scaffold with e.g. nbdev-new or ship-new, or remove): {', '.join(bad)}") return up = upgrade or _should_upgrade(root) @@ -793,6 +847,7 @@ async def ws_sync( _sync_cargo_keys(root) subprocess.run(["uv", "sync", "-U"] if up else ["uv", "sync"], check=True, cwd=root) if up: _upgrade_stamp(root).touch() + if js_members and (built := _sync_js(root, js_members)): print(f"JS build scripts run: {', '.join(os.path.relpath(d, root) for d in built)}") @call_parse async def ws_add( diff --git a/fastws/releases.py b/fastws/releases.py index db67f63..850b848 100644 --- a/fastws/releases.py +++ b/fastws/releases.py @@ -10,21 +10,12 @@ from ghapi.core import APIError, GhApi, dep_closure, local_dep_graph from packaging.version import InvalidVersion, Version -from .core import _load_repo_entries, _resolve_path, _ws_root - -try: import tomllib -except ModuleNotFoundError: import tomli as tomllib +from .core import _fastws_cfg, _load_repo_entries, _resolve_path, _ws_root # Start-of-message regexes for commits that need no release (curation noise, CI config, docs regen) DEFAULT_SKIP = ["bump$", "Bump version to ", "meta", "auto", "README", "regen readme", "nbdev regen", "update .gitignore", r"\.?gitignore$", "workflow", "chkstyle skip", "tests$", "md$", "docs?$", "clean$", "CI$", "ignore$", "allowed_metadata_keys$"] -def _fastws_cfg(root: Path) -> dict: - "The `[tool.fastws]` table from the workspace root pyproject.toml (empty when absent)." - pyproj = root/"pyproject.toml" - if not pyproj.exists(): return {} - return tomllib.loads(pyproj.read_text(encoding="utf-8")).get("tool", {}).get("fastws", {}) - def _skip_pats(skip=None, root: Path|None = None) -> list[re.Pattern]: "Compiled skip patterns: defaults + `[tool.fastws].release_skip` from the workspace pyproject + `skip`." pats = list(DEFAULT_SKIP) diff --git a/tests/test_sync.py b/tests/test_sync.py index aeda45e..a5f36bc 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,4 +1,4 @@ -import os, pytest +import json, os, pytest from types import SimpleNamespace from fastgit import Git import fastws.core as core @@ -207,6 +207,15 @@ def test_external_projects_discovers_root_and_subdir_packages(tmp_path): assert core._external_projects(root, [single, multi, tmp_path/'missing']) == [('singlepkg', '../single'), ('tool1', '../multi/tool1')] +@pytest.mark.parametrize('manifest', ['pyproject.toml', 'Cargo.toml', 'package.json']) +def test_project_dirs_expands_patterns_and_filters_manifests(tmp_path, manifest): + for name in ('a', 'b', 'empty'): (tmp_path/'pkgs'/name).mkdir(parents=True) + for name in ('a', 'b'): (tmp_path/'pkgs'/name/manifest).write_text('') + members, exclude = ['pkgs/*', 'pkgs/a'], ['./pkgs/b'] + assert core._project_dirs(tmp_path, manifest, members, exclude) == [tmp_path/'pkgs'/'a'] + assert core._project_dirs(tmp_path, members=members, exclude=exclude) == [tmp_path/'pkgs'/'a', tmp_path/'pkgs'/'empty'] + + def test_ws_projects_skip_excluded_dirs_and_template_names(tmp_path): (tmp_path/'pyproject.toml').write_text('[tool.uv.workspace]\nmembers = ["./*"]\nexclude = ["skip-*"]\n') for name, pkg in (('keep', 'keepme'), ('skip-template', 'skipme'), ('template', '{repo}')): @@ -454,3 +463,101 @@ def test_sync_cargo_wrapper(tmp_path, monkeypatch): config.write_text('[build]\nrustc-wrapper = "other-cache"\n') assert not core._sync_cargo_wrapper(tmp_path) assert core.tomllib.loads(config.read_text())['build']['rustc-wrapper'] == 'other-cache' + + +def test_npm_dirs_discovers_root_and_declared_packages(tmp_path): + for name in ('app', 'crate', 'crate/wasm', 'crate/frontend', 'crate/docs', '_scratch', 'node_modules', 'crate/node_modules'): + d = tmp_path/name + d.mkdir(parents=True, exist_ok=True) + (d/'package.json').write_text('{}') + (tmp_path/'app'/'package-lock.json').write_text('{}') + (tmp_path/'crate'/'package.json').write_text('{"private": true, "workspaces": ["wasm", "front*", "node_modules", "wasm", "missing"]}') + (tmp_path/'crate'/'frontend'/'bun.lock').write_text('') # lockfiles do not override explicit membership + + assert core._npm_dirs(tmp_path) == [tmp_path/'app', tmp_path/'crate', tmp_path/'crate'/'wasm', tmp_path/'crate'/'frontend'] + + +def test_npm_dirs_honours_fastws_exclude(tmp_path): + (tmp_path/'pyproject.toml').write_text('[tool.fastws]\nexclude = ["app", "tool*", "py/examples"]\n') + for name in ('app', 'lib', 'tools'): + (tmp_path/name).mkdir() + (tmp_path/name/'package.json').write_text('{}') + for name in ('examples', 'wasm'): + (tmp_path/'py'/name).mkdir(parents=True) + (tmp_path/'py'/name/'package.json').write_text('{}') # examples is excluded by its root-relative path + (tmp_path/'py'/'package.json').write_text('{"private": true, "workspaces": ["*"]}') + + assert core._npm_dirs(tmp_path) == [tmp_path/'lib', tmp_path/'py', tmp_path/'py'/'wasm'] + + +def test_sync_ws_package_json_generates_and_preserves(tmp_path): + pkg = tmp_path/'package.json' + pkg.write_text('{\n "name": "ws",\n "private": true,\n "workspaces": ["../outside", "gone", "tools/*"]\n}\n') + members = [tmp_path/'app', tmp_path/'crate'/'wasm'] + + added, removed = core._sync_ws_package_json(tmp_path, members) + data = json.loads(pkg.read_text()) + # kept: entries outside the root and globs; managed: existing dirs regenerated from discovery + assert data['workspaces'] == ['../outside', 'tools/*', 'app', 'crate/wasm'] + assert data['name'] == 'ws' and data['private'] is True # other keys untouched + assert added == ['app', 'crate/wasm'] and removed == ['gone'] + + content = pkg.read_text() + assert core._sync_ws_package_json(tmp_path, members) == ([], []) + assert pkg.read_text() == content + + # a missing root package.json is created + bare = tmp_path/'ws2' + bare.mkdir() + assert core._sync_ws_package_json(bare, [bare/'a']) == (['a'], []) + assert json.loads((bare/'package.json').read_text()) == {'private': True, 'workspaces': ['a']} + + +@pytest.mark.parametrize('workspaces', [{'packages': ['packages/*']}, {}]) +def test_sync_ws_package_json_rejects_object_form(tmp_path, workspaces): + pkg = tmp_path/'package.json' + content = json.dumps({'private': True, 'workspaces': workspaces}) + '\n' + pkg.write_text(content) + + with pytest.raises(SystemExit, match='workspaces.*list') as exc: core._sync_ws_package_json(tmp_path, [tmp_path/'app']) + assert str(pkg) in str(exc.value) + assert pkg.read_text() == content + + +def test_ws_excludes_treat_npm_only_dirs_like_cargo_only(tmp_path): + pyproject = tmp_path/'pyproject.toml' + pyproject.write_text('[project]\nname = "uvws"\n\n[tool.uv.workspace]\nmembers = ["./*"]\nexclude = []\n') + for name in ('app', 'pending'): (tmp_path/name).mkdir() + (tmp_path/'app'/'package.json').write_text('{}') + + # a tracked npm-only checkout is a valid JS project: excluded from uv, never pending; an empty tracked dir still awaits scaffolding + assert core._sync_ws_excludes(pyproject, tmp_path, {'app', 'pending'}) == (['app'], []) + assert core._pending_dirs(tmp_path) == ['pending'] + + +@pytest.mark.parametrize('tool', ['npm', 'custom-js']) +def test_sync_js_installs_then_builds_native_members(tmp_path, monkeypatch, tool): + if tool != 'npm': (tmp_path/'pyproject.toml').write_text(f'[tool.fastws]\njs = "{tool}"\n') + app = tmp_path/'app' + app.mkdir() + (app/'package.json').write_text('{"scripts": {"build": "vite build"}}') + crate = tmp_path/'crate' + wasm = crate/'wasm' + (wasm/'pkg').mkdir(parents=True) + (wasm/'pkg'/'out.wasm').write_text('') + for d in (crate, wasm): (d/'Cargo.toml').write_text('') + (crate/'package.json').write_text('{"private": true, "workspaces": ["wasm"]}') + (wasm/'package.json').write_text('{"scripts": {"build": "cargo build"}}') + calls = [] + monkeypatch.setattr(core.subprocess, 'run', lambda cmd, **kw: calls.append((cmd, kw.get('cwd')))) + members = [app, crate, wasm] + + # a tool that is not installed stops the sync with a one-line message before anything runs + monkeypatch.setattr(core.shutil, 'which', lambda t: None) + with pytest.raises(SystemExit, match='not installed'): core._sync_js(tmp_path, members) + assert calls == [] + monkeypatch.setattr(core.shutil, 'which', lambda t: f'/usr/bin/{t}') + + # Install first, then run the native build even with existing output. + assert core._sync_js(tmp_path, members) == [wasm] + assert calls == [([tool, 'install'], tmp_path), ([tool, 'run', 'build'], wasm)]