From 71bbc6c6abcf37523d000edb774b210f08b2c8ab Mon Sep 17 00:00:00 2001 From: Goutam Adwant <8672451+goutamadwant@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:03:44 -0700 Subject: [PATCH] feat: support cloud filesystem model paths Signed-off-by: Goutam Adwant <8672451+goutamadwant@users.noreply.github.com> --- CHANGELOG.md | 1 + README.md | 32 +++ pyproject.toml | 7 + src/model_signing/_cli.py | 77 +++++-- src/model_signing/_filesystem.py | 83 +++++++ src/model_signing/_hashing/io.py | 26 ++- src/model_signing/_serialization/file.py | 30 ++- .../_serialization/file_shard.py | 40 ++-- .../_serialization/serialization.py | 12 +- src/model_signing/hashing.py | 25 +- src/model_signing/verifying.py | 4 +- tests/cloud_filesystem_test.py | 218 ++++++++++++++++++ 12 files changed, 465 insertions(+), 90 deletions(-) create mode 100644 src/model_signing/_filesystem.py create mode 100644 tests/cloud_filesystem_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index db697efb..40d90895 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All versions prior to 1.0.0 are untracked. ### Added - Added the `digest` subcommand to compute and print a model's digest. This enables other tools to easily pair the attestations with a model directory. - Added `--module-paths` option to PKCS #11 signing methods pkcs11-key and pkcs11-certificate. +- Added streaming hashing, signing, and verification for models stored at GCS and S3 URIs, with provider-specific optional dependencies. ([#148](https://github.com/sigstore/model-transparency/issues/148)) ### Changed - Standardized CLI flags to use hyphens (e.g., `--trust-config` instead of `--trust_config`). Underscore variants are still accepted for backwards compatibility via token normalization. diff --git a/README.md b/README.md index f1417a21..30bb86dc 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,38 @@ subcommand, pointing it to the model directory: The digest subcommand follows the same ignore rules used when signing. +### Cloud-backed models + +Model paths can be GCS or S3 URIs. Install the extra for the provider that +stores the model: + +```bash +pip install "model-signing[gcs]" # gs:// +pip install "model-signing[s3]" # s3:// +``` + +The model URI can then be used anywhere a local model path is accepted: + +```bash +model_signing digest gs://my-bucket/models/bert +model_signing sign key gs://my-bucket/models/bert \ + --private-key key.priv --signature model.sig +model_signing verify key gs://my-bucket/models/bert \ + --public-key key.pub --signature model.sig +``` + +Model files are opened through the cloud filesystem backend and read in +bounded chunks. Shard serialization uses seek/range reads, so +`model-signing` does not stage the complete model on local disk. Any caching +performed by the installed provider is controlled by that provider. In this +scope, signature bundles, keys, certificates, and trust configuration files +remain local paths. + +Provider credentials use the standard `gcsfs` or `s3fs` configuration. The +test suite validates the URI, traversal, streaming, signing, and verification +paths with an in-memory `fsspec` backend; it does not exercise live cloud +credentials. + ## Using Private Sigstore Instances To use a private Sigstore setup (e.g. custom Rekor/Fulcio), use the `--trust-config` flag: diff --git a/pyproject.toml b/pyproject.toml index 6d6b4288..64a62d39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "blake3", "click", "cryptography", + "etils[epath]>=1.13", "in-toto-attestation", "sigstore>=4.0", "sigstore-models>=0.0.5", @@ -49,6 +50,12 @@ keywords = [ ] [project.optional-dependencies] +gcs = [ + "gcsfs", +] +s3 = [ + "s3fs", +] pkcs11 = [ "PyKCS11", ] diff --git a/src/model_signing/_cli.py b/src/model_signing/_cli.py index 5d529058..53b69b57 100644 --- a/src/model_signing/_cli.py +++ b/src/model_signing/_cli.py @@ -23,6 +23,7 @@ import click import model_signing +from model_signing import _filesystem class NoOpTracer: @@ -44,7 +45,7 @@ def set_attribute(self, key, value): # Decorator for the commonly used argument for the model path. _model_path_argument = click.argument( - "model_path", type=pathlib.Path, metavar="MODEL_PATH" + "model_path", type=str, metavar="MODEL_PATH" ) @@ -78,7 +79,7 @@ def set_attribute(self, key, value): # Decorator for the commonly used option to ignore certain paths _ignore_paths_option = click.option( "--ignore-paths", - type=pathlib.Path, + type=str, metavar="IGNORE_PATHS", multiple=True, help="File paths to ignore when signing or verifying.", @@ -166,13 +167,39 @@ def set_attribute(self, key, value): def _resolve_ignore_paths( - model_path: pathlib.Path, paths: Iterable[pathlib.Path] -) -> list[pathlib.Path]: - model_root = model_path.resolve() + model_path: model_signing.hashing.PathLike, + paths: Iterable[str | pathlib.Path], +) -> list[_filesystem.Path]: + model_root = _filesystem.as_path(model_path) + + if _filesystem.is_remote(model_root): + remote_paths = [] + for p in paths: + # Signature paths are local pathlib values. Explicit ignore paths + # arrive as strings and are resolved against the remote model. + if not isinstance(p, str): + continue + candidate = _filesystem.as_path(p) + if _filesystem.is_remote(candidate): + full = candidate + elif candidate.is_absolute(): + continue + else: + full = model_root / candidate + try: + remote_paths.append(full.relative_to(model_root)) + except ValueError: + continue + return remote_paths + + model_root = pathlib.Path(model_path).resolve() cwd = pathlib.Path.cwd() - resolved_paths = [] + resolved_paths: list[_filesystem.Path] = [] for p in paths: - candidate = (p if p.is_absolute() else (cwd / p)).resolve() + local_path = pathlib.Path(p) + candidate = ( + local_path if local_path.is_absolute() else (cwd / local_path) + ).resolve() try: resolved_paths.append(candidate.relative_to(model_root)) except ValueError: @@ -281,8 +308,8 @@ def main(log_level: str) -> None: @_ignore_git_paths_option @_allow_symlinks_option def _digest( - model_path: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + model_path: str, + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, ) -> None: @@ -380,8 +407,8 @@ def _sign() -> None: help="The custom OpenID Connect client secret to use during OAuth2", ) def _sign_sigstore( - model_path: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + model_path: str, + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, signature: pathlib.Path, @@ -470,8 +497,8 @@ def _sign_sigstore( help="Password for the key encryption, if any", ) def _sign_private_key( - model_path: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + model_path: str, + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, signature: pathlib.Path, @@ -518,8 +545,8 @@ def _sign_private_key( @_pkcs11_uri_option @_module_paths_option def _sign_pkcs11_key( - model_path: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + model_path: str, + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, signature: pathlib.Path, @@ -570,8 +597,8 @@ def _sign_pkcs11_key( @_signing_certificate_option @_certificate_root_of_trust_option def _sign_certificate( - model_path: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + model_path: str, + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, signature: pathlib.Path, @@ -626,8 +653,8 @@ def _sign_certificate( @_certificate_root_of_trust_option @_module_paths_option def _sign_pkcs11_certificate( - model_path: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + model_path: str, + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, signature: pathlib.Path, @@ -724,9 +751,9 @@ def _verify() -> None: ) @_ignore_unsigned_files_option def _verify_sigstore( - model_path: pathlib.Path, + model_path: str, signature: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, identity: str, @@ -792,9 +819,9 @@ def _verify_sigstore( ) @_ignore_unsigned_files_option def _verify_private_key( - model_path: pathlib.Path, + model_path: str, signature: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, public_key: pathlib.Path, @@ -865,9 +892,9 @@ def _verify_private_key( ) @_ignore_unsigned_files_option def _verify_certificate( - model_path: pathlib.Path, + model_path: str, signature: pathlib.Path, - ignore_paths: Iterable[pathlib.Path], + ignore_paths: Iterable[str], ignore_git_paths: bool, allow_symlinks: bool, certificate_chain: Iterable[pathlib.Path], diff --git a/src/model_signing/_filesystem.py b/src/model_signing/_filesystem.py new file mode 100644 index 00000000..e86862e9 --- /dev/null +++ b/src/model_signing/_filesystem.py @@ -0,0 +1,83 @@ +# Copyright 2026 The Sigstore Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Filesystem path helpers shared by model serialization code.""" + +from collections.abc import Iterator +import os +import pathlib +from typing import TypeAlias + +from etils import epath + + +PathLike: TypeAlias = str | bytes | os.PathLike +Path: TypeAlias = pathlib.Path | epath.Path + + +def as_path(path: PathLike) -> Path: + """Builds a local pathlib path or a URI-aware epath path.""" + if isinstance(path, (pathlib.Path, epath.Path)): + return path + + raw_path = os.fspath(path) + if isinstance(raw_path, bytes): + return pathlib.Path(os.fsdecode(raw_path)) + if "://" in raw_path: + return epath.Path(raw_path) + return pathlib.Path(raw_path) + + +def is_remote(path: Path) -> bool: + """Returns whether a path uses a non-local URI scheme.""" + return not isinstance(path, pathlib.Path) and "://" in os.fspath(path) + + +def is_symlink(path: Path) -> bool: + """Checks local symlinks; object-store paths cannot be symlinks.""" + if isinstance(path, pathlib.Path): + return path.is_symlink() + return False + + +def file_size(path: Path) -> int: + """Returns a file size for pathlib and epath stat result types.""" + result = path.stat() + if isinstance(result, os.stat_result): + return result.st_size + return result.length + + +def walk_paths(model_path: Path) -> Iterator[Path]: + """Yields a model and all of its descendants. + + pathlib's existing recursive glob behavior is retained for local paths. + etils intentionally rejects recursive glob patterns for cloud paths, so + remote directories are traversed explicitly through their path interface. + """ + yield model_path + if isinstance(model_path, pathlib.Path): + yield from model_path.glob("**/*") + return + + if not model_path.is_dir(): + return + + directories = [model_path] + while directories: + directory = directories.pop() + for child in directory.iterdir(): + yield child + if child.is_dir(): + directories.append(child) diff --git a/src/model_signing/_hashing/io.py b/src/model_signing/_hashing/io.py index dd963e92..7fcc1078 100644 --- a/src/model_signing/_hashing/io.py +++ b/src/model_signing/_hashing/io.py @@ -35,11 +35,10 @@ ``` """ -import pathlib - import blake3 from typing_extensions import override +from model_signing import _filesystem from model_signing._hashing import hashing @@ -65,7 +64,7 @@ class SimpleFileHasher(FileHasher): def __init__( self, - file: pathlib.Path, + file: _filesystem.Path, content_hasher: hashing.StreamingHashEngine, *, chunk_size: int = 1_048_576, @@ -93,7 +92,7 @@ def __init__( self._chunk_size = chunk_size self._digest_name_override = digest_name_override - def set_file(self, file: pathlib.Path) -> None: + def set_file(self, file: _filesystem.Path) -> None: """Redefines the file to be hashed in `compute`. Args: @@ -116,10 +115,10 @@ def compute(self) -> hashing.Digest: self._content_hasher.reset() if self._chunk_size == 0: - with open(self._file, "rb") as f: + with self._file.open("rb") as f: self._content_hasher.update(f.read()) else: - with open(self._file, "rb") as f: + with self._file.open("rb") as f: while True: data = f.read(self._chunk_size) if not data: @@ -145,7 +144,7 @@ class Blake3FileHasher(FileHasher): def __init__( self, - file: pathlib.Path, + file: _filesystem.Path, *, max_threads: int = blake3.blake3.AUTO, digest_name_override: str | None = None, @@ -163,7 +162,7 @@ def __init__( self._digest_name_override = digest_name_override self._blake3 = blake3.blake3(max_threads=max_threads) - def set_file(self, file: pathlib.Path) -> None: + def set_file(self, file: _filesystem.Path) -> None: """Redefines the file to be hashed in `compute`. Args: @@ -181,7 +180,12 @@ def digest_name(self) -> str: @override def compute(self) -> hashing.Digest: self._blake3.reset() - self._blake3.update_mmap(self._file) + if _filesystem.is_remote(self._file): + with self._file.open("rb") as f: + while data := f.read(1_048_576): + self._blake3.update(data) + else: + self._blake3.update_mmap(self._file) return hashing.Digest(self.digest_name, self._blake3.digest()) @property @@ -202,7 +206,7 @@ class ShardedFileHasher(SimpleFileHasher): def __init__( self, - file: pathlib.Path, + file: _filesystem.Path, content_hasher: hashing.StreamingHashEngine, *, start: int, @@ -276,7 +280,7 @@ def set_shard(self, *, start: int, end: int) -> None: def compute(self) -> hashing.Digest: self._content_hasher.reset() - with open(self._file, "rb") as f: + with self._file.open("rb") as f: f.seek(self._start) to_read = self._end - self._start if self._chunk_size == 0 or self._chunk_size >= to_read: diff --git a/src/model_signing/_serialization/file.py b/src/model_signing/_serialization/file.py index a6228679..f4df85d9 100644 --- a/src/model_signing/_serialization/file.py +++ b/src/model_signing/_serialization/file.py @@ -16,12 +16,12 @@ from collections.abc import Callable, Iterable import concurrent.futures -import itertools import os import pathlib from typing_extensions import override +from model_signing import _filesystem from model_signing import manifest from model_signing._hashing import io from model_signing._serialization import serialization @@ -36,7 +36,7 @@ class Serializer(serialization.Serializer): def __init__( self, - file_hasher_factory: Callable[[pathlib.Path], io.FileHasher], + file_hasher_factory: Callable[[_filesystem.Path], io.FileHasher], *, max_workers: int | None = None, allow_symlinks: bool = False, @@ -78,10 +78,10 @@ def set_allow_symlinks(self, allow_symlinks: bool) -> None: @override def serialize( self, - model_path: pathlib.Path, + model_path: _filesystem.Path, *, - ignore_paths: Iterable[pathlib.Path] = frozenset(), - files_to_hash: Iterable[pathlib.Path] | None = None, + ignore_paths: Iterable[_filesystem.Path] = frozenset(), + files_to_hash: Iterable[_filesystem.Path] | None = None, ) -> manifest.Manifest: """Serializes the model given by the `model_path` argument. @@ -101,14 +101,8 @@ def serialize( was not initialized with `allow_symlinks=True`. """ paths = [] - # TODO: github.com/sigstore/model-transparency/issues/200 - When - # Python3.12 is the minimum supported version, the glob can be replaced - # with `pathlib.Path.walk` for a clearer interface, and some speed - # improvement. if files_to_hash is None: - files_to_hash = itertools.chain( - (model_path,), model_path.glob("**/*") - ) + files_to_hash = _filesystem.walk_paths(model_path) for path in files_to_hash: if serialization.should_ignore(path, ignore_paths): continue @@ -134,10 +128,12 @@ def serialize( if ignore_paths: rel_ignore_paths = [] for p in ignore_paths: - rp = os.path.relpath(p, model_path) - # rp may start with "../" if it is not relative to model_path - if not rp.startswith("../"): - rel_ignore_paths.append(pathlib.Path(rp)) + try: + rel_ignore_paths.append( + pathlib.PurePosixPath(p.relative_to(model_path)) + ) + except ValueError: + continue hasher = self._hasher_factory(pathlib.Path()) self._serialization_description = manifest._FileSerialization( @@ -155,7 +151,7 @@ def serialize( ) def _compute_hash( - self, model_path: pathlib.Path, path: pathlib.Path + self, model_path: _filesystem.Path, path: _filesystem.Path ) -> manifest.FileManifestItem: """Produces the manifest item of the file given by `path`. diff --git a/src/model_signing/_serialization/file_shard.py b/src/model_signing/_serialization/file_shard.py index fc55706e..ef484aa5 100644 --- a/src/model_signing/_serialization/file_shard.py +++ b/src/model_signing/_serialization/file_shard.py @@ -16,12 +16,12 @@ from collections.abc import Callable, Iterable import concurrent.futures -import itertools import os import pathlib from typing_extensions import override +from model_signing import _filesystem from model_signing import manifest from model_signing._hashing import io from model_signing._serialization import serialization @@ -59,7 +59,7 @@ class Serializer(serialization.Serializer): def __init__( self, sharded_hasher_factory: Callable[ - [pathlib.Path, int, int], io.ShardedFileHasher + [_filesystem.Path, int, int], io.ShardedFileHasher ], *, max_workers: int | None = None, @@ -113,10 +113,10 @@ def set_allow_symlinks(self, allow_symlinks: bool) -> None: @override def serialize( self, - model_path: pathlib.Path, + model_path: _filesystem.Path, *, - ignore_paths: Iterable[pathlib.Path] = frozenset(), - files_to_hash: Iterable[pathlib.Path] | None = None, + ignore_paths: Iterable[_filesystem.Path] = frozenset(), + files_to_hash: Iterable[_filesystem.Path] | None = None, ) -> manifest.Manifest: """Serializes the model given by the `model_path` argument. @@ -135,14 +135,8 @@ def serialize( was not initialized with `allow_symlinks=True`. """ shards = [] - # TODO: github.com/sigstore/model-transparency/issues/200 - When - # Python3.12 is the minimum supported version, the glob can be replaced - # with `pathlib.Path.walk` for a clearer interface, and some speed - # improvement. if files_to_hash is None: - files_to_hash = itertools.chain( - (model_path,), model_path.glob("**/*") - ) + files_to_hash = _filesystem.walk_paths(model_path) for path in files_to_hash: if serialization.should_ignore(path, ignore_paths): continue @@ -167,10 +161,12 @@ def serialize( if ignore_paths: rel_ignore_paths = [] for p in ignore_paths: - rp = os.path.relpath(p, model_path) - # rp may start with "../" if it is not relative to model_path - if not rp.startswith("../"): - rel_ignore_paths.append(pathlib.Path(rp)) + try: + rel_ignore_paths.append( + pathlib.PurePosixPath(p.relative_to(model_path)) + ) + except ValueError: + continue hasher = self._hasher_factory(pathlib.Path(), 0, 1) self._serialization_description = manifest._ShardSerialization( @@ -189,11 +185,11 @@ def serialize( ) def _get_shards( - self, path: pathlib.Path - ) -> list[tuple[pathlib.Path, int, int]]: + self, path: _filesystem.Path + ) -> list[tuple[_filesystem.Path, int, int]]: """Determines the shards of a given file path.""" shards = [] - path_size = path.stat().st_size + path_size = _filesystem.file_size(path) if path_size > 0: start = 0 for end in _endpoints(self._shard_size, path_size): @@ -202,7 +198,11 @@ def _get_shards( return shards def _compute_hash( - self, model_path: pathlib.Path, path: pathlib.Path, start: int, end: int + self, + model_path: _filesystem.Path, + path: _filesystem.Path, + start: int, + end: int, ) -> manifest.ShardedFileManifestItem: """Produces the manifest item of the file given by `path`. diff --git a/src/model_signing/_serialization/serialization.py b/src/model_signing/_serialization/serialization.py index ba47bb43..748df99c 100644 --- a/src/model_signing/_serialization/serialization.py +++ b/src/model_signing/_serialization/serialization.py @@ -16,13 +16,13 @@ import abc from collections.abc import Iterable -import pathlib +from model_signing import _filesystem from model_signing import manifest def check_file_or_directory( - path: pathlib.Path, *, allow_symlinks: bool = False + path: _filesystem.Path, *, allow_symlinks: bool = False ) -> None: """Checks that the given path is either a file or a directory. @@ -42,7 +42,7 @@ def check_file_or_directory( ValueError: The path is neither a file or a directory, or the path is a symlink and `allow_symlinks` is false. """ - if not allow_symlinks and path.is_symlink(): + if not allow_symlinks and _filesystem.is_symlink(path): raise ValueError( f"Cannot use '{path}' because it is a symlink. This" " behavior can be changed with `allow_symlinks`." @@ -56,7 +56,7 @@ def check_file_or_directory( def should_ignore( - path: pathlib.Path, ignore_paths: Iterable[pathlib.Path] + path: _filesystem.Path, ignore_paths: Iterable[_filesystem.Path] ) -> bool: """Determines if the provided path should be ignored during serialization. @@ -76,9 +76,9 @@ class Serializer(metaclass=abc.ABCMeta): @abc.abstractmethod def serialize( self, - model_path: pathlib.Path, + model_path: _filesystem.Path, *, - ignore_paths: Iterable[pathlib.Path] = frozenset(), + ignore_paths: Iterable[_filesystem.Path] = frozenset(), ) -> manifest.Manifest: """Serializes the model given by the `model_path` argument. diff --git a/src/model_signing/hashing.py b/src/model_signing/hashing.py index c897a297..3bd06342 100644 --- a/src/model_signing/hashing.py +++ b/src/model_signing/hashing.py @@ -55,6 +55,7 @@ import blake3 +from model_signing import _filesystem from model_signing import manifest from model_signing._hashing import hashing from model_signing._hashing import io @@ -150,7 +151,7 @@ def hash( # All paths in ``_ignored_paths`` are expected to be relative to the # model directory. Join them to ``model_path`` and ensure they do not # escape it. - model_path = pathlib.Path(model_path) + model_path = _filesystem.as_path(model_path) ignored_paths = [] for p in self._ignored_paths: full = model_path / p @@ -176,9 +177,13 @@ def hash( self._serializer.set_allow_symlinks(self._allow_symlinks) return self._serializer.serialize( - pathlib.Path(model_path), + model_path, ignore_paths=ignored_paths, - files_to_hash=files_to_hash, + files_to_hash=( + None + if files_to_hash is None + else [_filesystem.as_path(p) for p in files_to_hash] + ), ) def _build_stream_hasher( @@ -210,7 +215,7 @@ def _build_file_hasher_factory( hashing_algorithm: Literal["sha256", "blake2", "blake3"] = "sha256", chunk_size: int = 1048576, max_workers: int | None = None, - ) -> Callable[[pathlib.Path], io.FileHasher]: + ) -> Callable[[_filesystem.Path], io.FileHasher]: """Builds the hasher factory for a serialization by file. Args: @@ -228,7 +233,7 @@ def _build_file_hasher_factory( if max_workers is None: max_workers = blake3.blake3.AUTO - def _factory(path: pathlib.Path) -> io.FileHasher: + def _factory(path: _filesystem.Path) -> io.FileHasher: if hashing_algorithm == "blake3": return io.Blake3FileHasher(path, max_threads=max_workers) hasher = self._build_stream_hasher(hashing_algorithm) @@ -241,7 +246,7 @@ def _build_sharded_file_hasher_factory( hashing_algorithm: Literal["sha256", "blake2"] = "sha256", chunk_size: int = 1048576, shard_size: int = 1_000_000_000, - ) -> Callable[[pathlib.Path, int, int], io.ShardedFileHasher]: + ) -> Callable[[_filesystem.Path, int, int], io.ShardedFileHasher]: """Builds the hasher factory for a serialization by file shards. This is not recommended for BLAKE3 because it is not necessary. BLAKE3 @@ -260,7 +265,7 @@ def _build_sharded_file_hasher_factory( """ def _factory( - path: pathlib.Path, start: int, end: int + path: _filesystem.Path, start: int, end: int ) -> io.ShardedFileHasher: hasher = self._build_stream_hasher(hashing_algorithm) return io.ShardedFileHasher( @@ -397,7 +402,7 @@ def set_ignored_paths( """ # Preserve the user-provided relative paths; they are resolved against # the model directory later when hashing. - self._ignored_paths = frozenset(pathlib.Path(p) for p in paths) + self._ignored_paths = frozenset(_filesystem.as_path(p) for p in paths) self._ignore_git_paths = ignore_git_paths return self @@ -412,9 +417,9 @@ def add_ignored_paths( the model directory. """ newset = set(self._ignored_paths) - model_path = pathlib.Path(model_path) + model_path = _filesystem.as_path(model_path) for p in paths: - candidate = pathlib.Path(p) + candidate = _filesystem.as_path(p) full = model_path / candidate try: full.relative_to(model_path) diff --git a/src/model_signing/verifying.py b/src/model_signing/verifying.py index ecacefc7..53aa4140 100644 --- a/src/model_signing/verifying.py +++ b/src/model_signing/verifying.py @@ -42,6 +42,7 @@ import pathlib import sys +from model_signing import _filesystem from model_signing import hashing from model_signing import manifest from model_signing._signing import sign_certificate as certificate @@ -112,8 +113,9 @@ def verify( ) if self._ignore_unsigned_files: + model_root = _filesystem.as_path(model_path) files_to_hash = [ - model_path / rd.identifier + model_root / rd.identifier for rd in expected_manifest.resource_descriptors() ] else: diff --git a/tests/cloud_filesystem_test.py b/tests/cloud_filesystem_test.py new file mode 100644 index 00000000..64428762 --- /dev/null +++ b/tests/cloud_filesystem_test.py @@ -0,0 +1,218 @@ +# Copyright 2026 The Sigstore Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for hashing models through cloud filesystem paths.""" + +from collections.abc import Iterator +import pathlib + +from click.testing import CliRunner +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec +from etils import epath +from fsspec.implementations.memory import MemoryFileSystem +import pytest + +from model_signing import _cli +from model_signing import _filesystem +from model_signing import hashing +from model_signing import signing +from model_signing import verifying + + +class _FakeGcsFileSystem(MemoryFileSystem): + """An in-memory fsspec filesystem accepting gs:// paths.""" + + protocol = "gs" + + +@pytest.fixture +def fake_gcs(monkeypatch: pytest.MonkeyPatch) -> Iterator[_FakeGcsFileSystem]: + """Routes etils GCS operations to an isolated memory filesystem.""" + filesystem = _FakeGcsFileSystem() + filesystem.store.clear() + filesystem.pseudo_dirs[:] = [""] + monkeypatch.setattr(epath.gpath, "_is_tf_installed", lambda: False) + monkeypatch.setattr( + epath.backend.fsspec_backend, "_get_filesystem", lambda _: filesystem + ) + yield filesystem + filesystem.store.clear() + filesystem.pseudo_dirs[:] = [""] + + +def _populate_remote_model(filesystem: _FakeGcsFileSystem) -> None: + filesystem.makedirs("gs://bucket/model/nested") + filesystem.pipe("gs://bucket/model/weights.bin", b"0123456789") + filesystem.pipe("gs://bucket/model/nested/config.json", b'{"v": 1}') + + +def _populate_local_model(model: pathlib.Path) -> None: + (model / "nested").mkdir(parents=True) + (model / "weights.bin").write_bytes(b"0123456789") + (model / "nested/config.json").write_bytes(b'{"v": 1}') + + +@pytest.mark.parametrize( + "config", + [ + hashing.Config().use_file_serialization(chunk_size=3), + hashing.Config().use_file_serialization(hashing_algorithm="blake3"), + hashing.Config().use_shard_serialization( + chunk_size=2, shard_size=4, max_workers=1 + ), + ], +) +def test_remote_hash_matches_local( + config: hashing.Config, fake_gcs: _FakeGcsFileSystem, tmp_path: pathlib.Path +) -> None: + """Cloud and local paths produce the same canonical manifest.""" + _populate_remote_model(fake_gcs) + local_model = tmp_path / "model" + _populate_local_model(local_model) + + remote_manifest = config.hash("gs://bucket/model") + local_manifest = config.hash(local_model) + + assert remote_manifest == local_manifest + assert remote_manifest.model_name == local_manifest.model_name == "model" + assert ( + remote_manifest.serialization_type == local_manifest.serialization_type + ) + + +def test_remote_streaming_reads_are_bounded( + fake_gcs: _FakeGcsFileSystem, monkeypatch: pytest.MonkeyPatch +) -> None: + """File hashing requests bounded chunks instead of an unbounded read.""" + _populate_remote_model(fake_gcs) + requested_sizes = [] + original_open = fake_gcs.open + + class _RecordingReader: + def __init__(self, wrapped): + self._wrapped = wrapped + + def __enter__(self): + self._wrapped.__enter__() + return self + + def __exit__(self, *args): + return self._wrapped.__exit__(*args) + + def __getattr__(self, name): + return getattr(self._wrapped, name) + + def read(self, size=-1): + requested_sizes.append(size) + return self._wrapped.read(size) + + def recording_open(path, mode="rb", **kwargs): + opened = original_open(path, mode=mode, **kwargs) + if "r" in mode: + return _RecordingReader(opened) + return opened + + monkeypatch.setattr(fake_gcs, "open", recording_open) + + hashing.Config().use_file_serialization(chunk_size=3, max_workers=1).hash( + "gs://bucket/model" + ) + + assert requested_sizes + assert -1 not in requested_sizes + assert max(requested_sizes) == 3 + + +def test_remote_ignore_paths_are_model_relative( + fake_gcs: _FakeGcsFileSystem, +) -> None: + """Remote ignore paths stay within the model and omit matching files.""" + _populate_remote_model(fake_gcs) + ignore_paths = _cli._resolve_ignore_paths( + "gs://bucket/model", + ["nested/config.json", "gs://another-bucket/outside"], + ) + + manifest = ( + hashing.Config() + .set_ignored_paths(paths=ignore_paths, ignore_git_paths=False) + .hash("gs://bucket/model") + ) + + assert [str(path) for path in ignore_paths] == ["nested/config.json"] + assert [ + descriptor.identifier for descriptor in manifest.resource_descriptors() + ] == ["weights.bin"] + + +def test_sign_and_verify_remote_model( + fake_gcs: _FakeGcsFileSystem, tmp_path: pathlib.Path +) -> None: + """Signing and verification accept the same remote model URI.""" + _populate_remote_model(fake_gcs) + private_key = ec.generate_private_key(ec.SECP256R1()) + private_key_path = tmp_path / "key.pem" + public_key_path = tmp_path / "key.pub" + signature_path = tmp_path / "model.sig" + private_key_path.write_bytes( + private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + public_key_path.write_bytes( + private_key.public_key().public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + ) + + signing.Config().use_elliptic_key_signer(private_key=private_key_path).sign( + "gs://bucket/model", signature_path + ) + + verifier = verifying.Config().use_elliptic_key_verifier( + public_key=public_key_path + ) + verifier.verify("gs://bucket/model", signature_path) + + fake_gcs.pipe("gs://bucket/model/unsigned.txt", b"not signed") + verifier.set_ignore_unsigned_files(True).verify( + "gs://bucket/model", signature_path + ) + + fake_gcs.pipe("gs://bucket/model/weights.bin", b"tampered") + with pytest.raises(ValueError, match="Signature mismatch"): + verifier.verify("gs://bucket/model", signature_path) + + +def test_digest_cli_preserves_remote_uri(fake_gcs: _FakeGcsFileSystem) -> None: + """The CLI does not collapse the double slash in a cloud URI.""" + _populate_remote_model(fake_gcs) + result = CliRunner().invoke(_cli.main, ["digest", "gs://bucket/model"]) + + assert result.exit_code == 0, result.output + algorithm, digest = result.output.strip().split(":") + assert algorithm == "sha256" + assert len(digest) == 64 + + +def test_path_conversion_preserves_local_pathlib() -> None: + """Existing pathlib objects retain their exact local path behavior.""" + local_path = pathlib.Path("model") + + assert _filesystem.as_path(local_path) is local_path + assert str(_filesystem.as_path("gs://bucket/model")) == "gs://bucket/model"