diff --git a/.gitignore b/.gitignore index 873a5132b8..afce5d8dc1 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,7 @@ web_modules/ .netlify /packages/frontend/docs/ /target/ +/build.ninja # Sentry files instrument.mjs diff --git a/build-targets.json b/build-targets.json new file mode 100644 index 0000000000..f4bb7505d4 --- /dev/null +++ b/build-targets.json @@ -0,0 +1,50 @@ +{ + "document-types-wasm": { + "crate": "packages/document-types", + "command": "wasm-pack build packages/document-types", + "inputs": [ + "Cargo.lock", + "Cargo.toml", + "rust-toolchain.toml", + "packages/document-types/.cargo/config.toml", + "packages/document-types/Cargo.toml", + "packages/document-types/README.md", + "packages/document-types/src" + ], + "outputs": [ + "packages/document-types/pkg/package.json", + "packages/document-types/pkg/catcolab_document_types.js", + "packages/document-types/pkg/catcolab_document_types_bg.js", + "packages/document-types/pkg/catcolab_document_types.d.ts", + "packages/document-types/pkg/catcolab_document_types_bg.wasm", + "packages/document-types/pkg/catcolab_document_types_bg.wasm.d.ts", + "packages/document-types/pkg/README.md" + ] + }, + "catlog-wasm-browser": { + "crate": "packages/catlog-wasm", + "command": "wasm-pack build packages/catlog-wasm --target browser -d dist/pkg-browser --debug", + "inputs": [ + "Cargo.lock", + "Cargo.toml", + "rust-toolchain.toml", + "packages/catlog/Cargo.toml", + "packages/catlog/examples", + "packages/catlog/src", + "packages/catlog-wasm/.cargo/config.toml", + "packages/catlog-wasm/Cargo.toml", + "packages/catlog-wasm/src", + "packages/document-types/.cargo/config.toml", + "packages/document-types/Cargo.toml", + "packages/document-types/src" + ], + "outputs": [ + "packages/catlog-wasm/dist/pkg-browser/package.json", + "packages/catlog-wasm/dist/pkg-browser/catlog_wasm.js", + "packages/catlog-wasm/dist/pkg-browser/catlog_wasm_bg.js", + "packages/catlog-wasm/dist/pkg-browser/catlog_wasm.d.ts", + "packages/catlog-wasm/dist/pkg-browser/catlog_wasm_bg.wasm", + "packages/catlog-wasm/dist/pkg-browser/catlog_wasm_bg.wasm.d.ts" + ] + } +} diff --git a/build.py b/build.py new file mode 100755 index 0000000000..c75735d5fd --- /dev/null +++ b/build.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +from io import StringIO +from pathlib import Path +import json +import os + +import ninja +from ninja.ninja_syntax import Writer + + +ROOT = Path(__file__).resolve().parent +MANIFEST = ROOT / "build-targets.json" +BUILD_FILE = ROOT / "build.ninja" +BUILD_DIR = "target/ninja" + +# Inputs shared by every target; changing the build definition rebuilds everything. +COMMON_INPUTS = ["build.py", "build-targets.json"] + + +def expand_inputs(inputs: list[str]) -> list[str]: + """Expand directory inputs to the files they contain, as ninja needs files.""" + files: set[Path] = set() + for rel in inputs: + path = ROOT / rel + if path.is_dir(): + files.update(p for p in path.rglob("*") if p.is_file()) + else: + files.add(path) + return sorted(p.relative_to(ROOT).as_posix() for p in files) + + +targets = json.loads(MANIFEST.read_text()) + +buffer = StringIO() +w = Writer(buffer) + +w.variable("builddir", BUILD_DIR) +w.newline() + +w.pool("wasm_pool", depth=1) +w.newline() + +w.rule( + "wasm_pack", + command=f"mkdir -p {BUILD_DIR} && $wasm_pack_command && touch $out", + description="WASM-PACK $target_name", + pool="wasm_pool", + restat=True, +) +w.newline() + +for name, target in targets.items(): + stamp = f"{BUILD_DIR}/{name}.stamp" + w.build( + stamp, + "wasm_pack", + inputs=expand_inputs(COMMON_INPUTS + target["inputs"]), + implicit_outputs=target["outputs"], + variables={ + "wasm_pack_command": target["command"], + "target_name": name, + }, + ) + w.build(name, "phony", stamp) + w.newline() + +w.build("wasm", "phony", list(targets)) +w.default("wasm") + +contents = buffer.getvalue() +if not BUILD_FILE.exists() or BUILD_FILE.read_text() != contents: + BUILD_FILE.write_text(contents) + +os.chdir(ROOT) +ninja.ninja() diff --git a/flake.nix b/flake.nix index 5606d923ab..ce4d0dea25 100644 --- a/flake.nix +++ b/flake.nix @@ -178,13 +178,14 @@ esbuild lld netcat + ninja nodejs_24 nix openssl pkg-config pnpm postgresql - python3 + (python3.withPackages (ps: [ ps.ninja ])) python312Packages.ipykernel python312Packages.jupyter-core python312Packages.jupyter-server @@ -286,8 +287,12 @@ in { catcolabApi = pkgs.callPackage ./infrastructure/catcolab-api.nix craneArgs; - catlog-wasm-browser = pkgs.callPackage ./packages/catlog-wasm/default.nix craneArgs; - document-types-wasm = pkgs.callPackage ./packages/document-types/default.nix craneArgs; + catlog-wasm-browser = pkgs.callPackage ./infrastructure/ninja-target.nix ( + craneArgs // { target = "catlog-wasm-browser"; } + ); + document-types-wasm = pkgs.callPackage ./infrastructure/ninja-target.nix ( + craneArgs // { target = "document-types-wasm"; } + ); frontend = frontendPackage.package; frontend-tests = frontendPackage.tests; }; diff --git a/infrastructure/ninja-target.nix b/infrastructure/ninja-target.nix new file mode 100644 index 0000000000..32066d9a4b --- /dev/null +++ b/infrastructure/ninja-target.nix @@ -0,0 +1,60 @@ +# Builds a target from build-targets.json via build.py (ninja). The source +# fileset is derived from the target's `inputs`, so ninja and nix share one +# dependency listing. +{ + craneLib, + cargoArtifacts, + pkgs, + target, +}: +let + root = ../.; + targets = builtins.fromJSON (builtins.readFile (root + "/build-targets.json")); + spec = targets.${target}; + toPath = rel: root + "/${rel}"; +in +craneLib.buildPackage { + inherit cargoArtifacts; + inherit (craneLib.crateNameFromCargoToml { cargoToml = toPath "${spec.crate}/Cargo.toml"; }) + version + pname + ; + doCheck = false; + + nativeBuildInputs = [ + (pkgs.python3.withPackages (ps: [ ps.ninja ])) + pkgs.wasm-pack + pkgs.wasm-bindgen-cli + pkgs.binaryen + pkgs.nodejs + ]; + + buildInputs = [ + pkgs.openssl + ]; + + src = pkgs.lib.fileset.toSource { + inherit root; + fileset = pkgs.lib.fileset.unions ( + map toPath ( + [ + "build.py" + "build-targets.json" + ] + ++ spec.inputs + ) + ); + }; + + # wasm-pack expects a wasm-bindgen-cli in the environment matching the version in Cargo.lock; + # see the overlay in flake.nix. It also needs a writeable $HOME (https://github.com/ipetkov/crane/issues/362). + buildPhase = '' + HOME=$(mktemp -d) python3 build.py ${target} + ''; + + installPhase = '' + mkdir -p $out + cp ${pkgs.lib.concatStringsSep " " spec.outputs} $out/ + ls $out/ + ''; +} diff --git a/packages/catlog-wasm/default.nix b/packages/catlog-wasm/default.nix deleted file mode 100644 index b4ff0f2743..0000000000 --- a/packages/catlog-wasm/default.nix +++ /dev/null @@ -1,59 +0,0 @@ -{ - craneLib, - cargoArtifacts, - pkgs, -}: -craneLib.buildPackage { - inherit cargoArtifacts; - inherit (craneLib.crateNameFromCargoToml { cargoToml = ./Cargo.toml; }) version pname; - doCheck = false; - - cargoExtraArgs = "-p catlog-wasm"; - - nativeBuildInputs = [ - pkgs.wasm-pack - pkgs.wasm-bindgen-cli - pkgs.binaryen - pkgs.nodejs - ]; - - buildInputs = [ - pkgs.openssl - ]; - - src = pkgs.lib.fileset.toSource { - root = ../..; - fileset = pkgs.lib.fileset.unions [ - ../../Cargo.toml - ../../Cargo.lock - (craneLib.fileset.commonCargoSources ./.) - (craneLib.fileset.commonCargoSources ../catlog) - (craneLib.fileset.commonCargoSources ../document-types) - ./package.json - ]; - }; - - # run wasm-pack instead of the default cargo - buildPhase = '' - cd packages/catlog-wasm - # Run the wasm-pack command. wasm-pack will expect to find version of wasm-bindgen-cli in the - # environment that must matches the version wasm-bindgen used in the Cargo.toml. The wasm-bindgen-cli - # in the nix environment is defined an overlay in flake.nix. - # - # If the versions do not match there will be a build error when building with nix: - # Error: Not able to find or install a local wasm-bindgen. - # - # With RUST_LOG=debug set there should be a log like indicating the exact problem: - # Checking installed `wasm-bindgen` version == expected version: 0.2.105 == 0.2.106 - # - # wasm-pack needs a writeable $HOME - # https://github.com/ipetkov/crane/issues/362 - HOME=$(mktemp -d) npm run build:browser - ''; - - installPhase = '' - mkdir -p $out - cp -r dist/pkg-browser/* $out/ - ls $out/ - ''; -} diff --git a/packages/catlog-wasm/package.json b/packages/catlog-wasm/package.json index 1a8046b398..fce3f49f75 100644 --- a/packages/catlog-wasm/package.json +++ b/packages/catlog-wasm/package.json @@ -5,6 +5,6 @@ "version": "0.1.0", "type": "module", "scripts": { - "build:browser": "wasm-pack build ./. --target browser -d ./dist/pkg-browser --debug" + "build:browser": "../../build.py catlog-wasm-browser" } } diff --git a/packages/document-methods/package.json b/packages/document-methods/package.json index a796f16cac..119670994e 100644 --- a/packages/document-methods/package.json +++ b/packages/document-methods/package.json @@ -9,7 +9,7 @@ "./*": "./src/*" }, "scripts": { - "build:wasm": "wasm-pack build ../document-types", + "build:wasm": "../../build.py document-types-wasm", "format": "oxfmt .", "lint": "oxlint --fix && oxfmt .", "check": "pnpm run build:wasm && tsc && pnpm run lint", diff --git a/packages/document-types/default.nix b/packages/document-types/default.nix deleted file mode 100644 index bb055271bd..0000000000 --- a/packages/document-types/default.nix +++ /dev/null @@ -1,56 +0,0 @@ -{ - craneLib, - cargoArtifacts, - pkgs, -}: -craneLib.buildPackage { - inherit cargoArtifacts; - inherit (craneLib.crateNameFromCargoToml { cargoToml = ./Cargo.toml; }) version pname; - doCheck = false; - - cargoExtraArgs = "-p catcolab-document-types"; - - nativeBuildInputs = [ - pkgs.wasm-pack - pkgs.wasm-bindgen-cli - pkgs.binaryen - pkgs.nodejs - ]; - - buildInputs = [ - pkgs.openssl - ]; - - src = pkgs.lib.fileset.toSource { - root = ../..; - fileset = pkgs.lib.fileset.unions [ - ../../Cargo.toml - ../../Cargo.lock - (craneLib.fileset.commonCargoSources ./.) - ]; - }; - - # run wasm-pack instead of the default cargo - buildPhase = '' - cd packages/document-types - # Run the wasm-pack command. wasm-pack will expect to find version of wasm-bindgen-cli in the - # environment that must matches the version wasm-bindgen used in the Cargo.toml. The wasm-bindgen-cli - # in the nix environment is defined an overlay in flake.nix. - # - # If the versions do not match there will be a build error when building with nix: - # Error: Not able to find or install a local wasm-bindgen. - # - # With RUST_LOG=debug set there should be a log like indicating the exact problem: - # Checking installed `wasm-bindgen` version == expected version: 0.2.105 == 0.2.106 - # - # wasm-pack needs a writeable $HOME - # https://github.com/ipetkov/crane/issues/362 - HOME=$(mktemp -d) wasm-pack build ./. -d ./pkg - ''; - - installPhase = '' - mkdir -p $out - cp -r pkg/* $out/ - ls $out/ - ''; -} diff --git a/packages/frontend/package.json b/packages/frontend/package.json index b6c35898f8..e278cef667 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -5,8 +5,8 @@ "license": "MIT", "type": "module", "scripts": { - "build:doc-types": "pnpm --prefix ../document-methods run build:wasm", - "build:wasm": "pnpm --prefix ../catlog-wasm run build:browser", + "build:doc-types": "../../build.py document-types-wasm", + "build:wasm": "../../build.py catlog-wasm-browser", "build:bindings": "cargo run -p backend -- generate-bindings", "build:tcm": "tcm -p 'src/**/*.module.css' . && pnpm --prefix ../ui-components run build:tcm", "build:deps": "pnpm run build:doc-types && pnpm run build:wasm && pnpm run build:bindings && pnpm run build:tcm",