diff --git a/README.md b/README.md index 95ed3ea..8b2da84 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,7 @@ poetry bundle venv .build/.venv --without dev package-python-function .build/.venv --output-dir .build/lambda ``` -The output will be a .zip file with the same name as your project from your `pyproject.toml` file (with dashes replaced -with underscores). +The output will be a .zip file named after your project, as described in [Output file name](#output-file-name). ## Installation Use [pipx](https://github.com/pypa/pipx) to install: @@ -27,16 +26,57 @@ pipx install package-python-function ## Usage / Arguments ```shell -package-python-function venv_dir [--project PROJECT] [--output-dir OUTPUT_DIR] [--output OUTPUT] +package-python-function venv_dir [--project PROJECT] [--output-dir OUTPUT_DIR] [--output OUTPUT] [--report REPORT] ``` - `venv_dir` [Required]: The path to the virtual environment to package. - `--project` [Optional]: Path to the `pyproject.toml` file. Omit to use the `pyproject.toml` file in the current working directory. +- `--report` [Optional]: Path to write a JSON [report file](#report-file) to. Omit to write no report. -One of the following must be specified: +`--output` and `--output-dir` cannot be used together. If neither is given, the zip is written to the current working +directory. - `--output`: The full output path of the final zip file. -- `--output-dir`: The output directory for the final zip file. The name of the zip file will be based on the project's -name in the `pyproject.toml` file (with dashes replaced with underscores). +- `--output-dir`: The output directory for the final zip file. The name of the zip file is described in +[Output file name](#output-file-name). + +## Output file name + +Unless `--output` gives an exact path, the file written is `/.zip`. + +`distribution_name` is the project's name — `[project].name`, or `[tool.poetry].name` if that is absent — with each run +of characters outside `A-Z a-z 0-9 _ .` replaced by a single underscore, following the +[PyPA escaping rules](https://peps.python.org/pep-0427/#escaping-and-unicode). + +**Case is preserved.** A project named `My-App` produces `My_App.zip`, not `my_app.zip`. Note that this differs from the +wheel your build tool produces for the same project, whose filename is lowercased — so a wheel's name is not a safe way +to predict the name of this file. + +## Report file + +Pass `--report ` to have the tool write a JSON description of what it produced, so a calling script does not have +to re-derive the output path or re-measure the package. + +```json +{ + "output_file": "/abs/path/lambda/my_app.zip", + "distribution_name": "my_app", + "output_bytes": 3460000, + "uncompressed_bytes": 412000000, + "compressed_bytes": 3456789, + "nested_zip": false +} +``` + +| Field | Meaning | +| --- | --- | +| `output_file` | Absolute path of the zip that was written. | +| `distribution_name` | The normalized project name, as described in [Output file name](#output-file-name). | +| `output_bytes` | Size of the file at `output_file`. This is the artifact you deploy. | +| `uncompressed_bytes` | Total size of the packaged files before compression. This is the figure compared against the AWS Lambda 250 MiB unzipped limit. | +| `compressed_bytes` | Size of the dependencies zip. Equal to `output_bytes` unless the nested-zip strategy was used, in which case the outer zip also holds the loader. | +| `nested_zip` | Whether the nested-zip strategy was used. | + +The report is written only when packaging succeeds, so its presence is a reliable signal that the zip is really there. ## Notes on Reproducibility diff --git a/package_python_function/main.py b/package_python_function/main.py index da26892..7c890d2 100644 --- a/package_python_function/main.py +++ b/package_python_function/main.py @@ -4,17 +4,24 @@ import sys from .packager import Packager +from .reproducible_zipfile import date_time def main() -> None: logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s") args = parse_args() + + # Validate $SOURCE_DATE_EPOCH here, so that a bad value fails before any packaging work rather than partway + # through writing the zip. + date_time() + project_path = Path(args.project).resolve() venv_path = Path(args.venv_dir).resolve() output_dir_path = Path(args.output_dir).resolve() output_file_path = Path(args.output).resolve() if args.output else None - packager = Packager(venv_path, project_path, output_dir_path, output_file_path) + report_file_path = Path(args.report).resolve() if args.report else None + packager = Packager(venv_path, project_path, output_dir_path, output_file_path, report_file_path) packager.package() @@ -22,6 +29,8 @@ def parse_args() -> argparse.Namespace: arg_parser = argparse.ArgumentParser() arg_parser.add_argument("venv_dir", type=str, help="The directory path to the virtual environment to package into a zip file") arg_parser.add_argument("--project", type=str, default='pyproject.toml', help="The path to the project's pyproject.toml file. Omit to use pyproject.toml in the current working directory.") - arg_parser.add_argument("--output-dir", type=str, default='.', help="The directory path to save the output zip file. Default is the current working directory.") - arg_parser.add_argument("--output", type=str, default='', help="The full file path for the output file. Use this instead of --output-dir if you want total control of the output file path.") + output_group = arg_parser.add_mutually_exclusive_group() + output_group.add_argument("--output-dir", type=str, default='.', help="The directory path to save the output zip file. Default is the current working directory.") + output_group.add_argument("--output", type=str, default='', help="The full file path for the output file. Use this instead of --output-dir if you want total control of the output file path.") + arg_parser.add_argument("--report", type=str, default='', help="The file path to write a JSON report of the packaging result to. Omit to write no report.") return arg_parser.parse_args() diff --git a/package_python_function/packager.py b/package_python_function/packager.py index a3e8ea7..c7091d4 100644 --- a/package_python_function/packager.py +++ b/package_python_function/packager.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import logging import shutil from pathlib import Path @@ -11,26 +12,52 @@ logger = logging.getLogger(__name__) +class PackageTooLargeError(Exception): + """Raise when the content is too large for AWS Lambda, both uncompressed and compressed""" + + def __init__(self, uncompressed_bytes: int, compressed_bytes: int, limit_bytes: int): + self.uncompressed_bytes = uncompressed_bytes + self.compressed_bytes = compressed_bytes + self.limit_bytes = limit_bytes + super().__init__( + f"The uncompressed size ({uncompressed_bytes:,} bytes) is too large for AWS Lambda, and the compressed " + f"size ({compressed_bytes:,} bytes) also exceeds the limit of {limit_bytes:,} bytes, so the nested-zip " + "strategy cannot be used either. No package was written." + ) + class Packager: AWS_LAMBDA_MAX_UNZIP_SIZE = 262_144_000 DIRS_TO_EXCLUDE = ["__pycache__"] DIST_INFO_FILES_TO_EXCLUDE = ["RECORD", "direct_url.json"] EXTENSIONS_TO_EXCLUDE = [".pyc", ".pyo"] - def __init__(self, venv_path: Path, project_path: Path, output_dir: Path, output_file: Path | None): + def __init__( + self, + venv_path: Path, + project_path: Path, + output_dir: Path, + output_file: Path | None, + report_file: Path | None = None, + ): self.project = PythonProject(project_path) self.venv_path = venv_path self.output_dir = output_file.parent if output_file else output_dir self.output_file = output_file if output_file else output_dir / f'{self.project.distribution_name}.zip' + self.report_file = report_file self._uncompressed_bytes = 0 + self._compressed_bytes = 0 + self._nested_zip = False @property def input_path(self) -> Path: python_paths = list((self.venv_path / 'lib').glob('python*')) if not python_paths: - raise Exception("input_path") + raise FileNotFoundError( + f"No 'lib/python*' directory was found in '{self.venv_path}'. Check that this path points at a " + "virtual environment with the function's dependencies installed into it." + ) return python_paths[0] / 'site-packages' def package(self) -> None: @@ -41,6 +68,27 @@ def package(self) -> None: with NamedTemporaryFile(suffix=".zip") as dependencies_zip: self.zip_all_dependencies(Path(dependencies_zip.name)) + if self.report_file: + self.write_report() + + def write_report(self) -> None: + """ + Write a JSON report describing the package that was just produced, so that a calling script does not have to + re-derive the output path or re-measure the sizes. + """ + report = { + "output_file": str(self.output_file), + "distribution_name": self.project.distribution_name, + "output_bytes": self.output_file.stat().st_size, + "uncompressed_bytes": self._uncompressed_bytes, + "compressed_bytes": self._compressed_bytes, + "nested_zip": self._nested_zip, + } + + logger.info(f"Writing report to '{self.report_file}'...") + self.report_file.parent.mkdir(parents=True, exist_ok=True) + self.report_file.write_text(json.dumps(report, indent=2) + "\n") + def zip_all_dependencies(self, target_path: Path) -> None: logger.info(f"Zipping to {target_path}...") @@ -64,7 +112,7 @@ def zip_dir(path: Path) -> None: zip_dir(self.input_path) - compressed_bytes = target_path.stat().st_size + compressed_bytes = self._compressed_bytes = target_path.stat().st_size logger.info(f"Uncompressed size: {self._uncompressed_bytes:,} bytes. Compressed size: {compressed_bytes:,} bytes.") @@ -72,9 +120,12 @@ def zip_dir(path: Path) -> None: logger.info(f"The uncompressed size of the ZIP file is greater than the AWS Lambda limit of {self.AWS_LAMBDA_MAX_UNZIP_SIZE:,} bytes.") if(compressed_bytes < self.AWS_LAMBDA_MAX_UNZIP_SIZE): logger.info(f"The compressed size ({compressed_bytes:,}) is less than the AWS limit, so the nested-zip strategy will be used.") + self._nested_zip = True self.generate_nested_zip(target_path) else: - print("TODO Error. The unzipped size it too large for AWS Lambda.") + raise PackageTooLargeError( + self._uncompressed_bytes, compressed_bytes, self.AWS_LAMBDA_MAX_UNZIP_SIZE + ) else: logger.info(f"Copying '{target_path}' to '{self.output_file}'") shutil.copy(str(target_path), str(self.output_file)) diff --git a/package_python_function/python_project.py b/package_python_function/python_project.py index f12b563..84f7121 100644 --- a/package_python_function/python_project.py +++ b/package_python_function/python_project.py @@ -25,7 +25,7 @@ def name(self) -> str: """ @cached_property def distribution_name(self) -> str: - return re.sub("[^\w\d.]+", "_", self.name, re.UNICODE) + return re.sub(r"[^\w\d.]+", "_", self.name) @cached_property def entrypoint_package_name(self) -> str: @@ -41,7 +41,8 @@ def find_value(self, paths: tuple[tuple[str]]) -> str: value = self.get_value(path) if value is not None: return value - raise Exception("TODO Exception find_value") + searched = ", ".join(".".join(path) for path in paths) + raise ValueError(f"None of the following were found in '{self.path}': {searched}.") def get_value(self, path: tuple[str]) -> Optional[str]: node = self.toml diff --git a/package_python_function/reproducible_zipfile.py b/package_python_function/reproducible_zipfile.py index 9336457..0535125 100644 --- a/package_python_function/reproducible_zipfile.py +++ b/package_python_function/reproducible_zipfile.py @@ -25,7 +25,14 @@ def date_time() -> Tuple[int, int, int, int, int, int]: """ source_date_epoch = os.environ.get("SOURCE_DATE_EPOCH", None) if source_date_epoch is not None: - dt = time.gmtime(int(source_date_epoch))[:6] + try: + seconds_since_epoch = int(source_date_epoch) + except ValueError as error: + raise SourceDateEpochError( + f"$SOURCE_DATE_EPOCH must be an integer number of seconds since the Epoch, but was " + f"'{source_date_epoch}'." + ) from error + dt = time.gmtime(seconds_since_epoch)[:6] if dt[0] < 1980: raise SourceDateEpochError( "$SOURCE_DATE_EPOCH must be >= 315532800, since ZIP files need MS-DOS date/time format, which can be 1/1/1980, at minimum." diff --git a/tests/test_package_python_function.py b/tests/test_package_python_function.py index 8965608..8596c3a 100644 --- a/tests/test_package_python_function.py +++ b/tests/test_package_python_function.py @@ -1,3 +1,4 @@ +import json import sys import zipfile from pathlib import Path @@ -6,12 +7,14 @@ from _pytest.monkeypatch import MonkeyPatch from package_python_function.main import main +from package_python_function.packager import PackageTooLargeError, Packager +from package_python_function.python_project import PythonProject from package_python_function.reproducible_zipfile import ( DEFAULT_DATE_TIME, SourceDateEpochError, ) -from .conftest import Data, verify_file_reproducibility +from .conftest import Data, File, verify_file_reproducibility @pytest.mark.parametrize( "src_epoch, expected_exception, expected_date_time", @@ -179,3 +182,241 @@ def test_package_python_function_nested( assert not (verify_dir / file.path).exists() else: assert (verify_dir / file.path).exists() + +def _expected_uncompressed_bytes(data: Data) -> int: + return sum( + len(file.contents.encode()) + for file in data.project_files + if file not in data.files_excluded_from_bundle + ) + +def test_report_describes_a_single_zip(test_data: Data, tmp_path: Path) -> None: + output_dir_path = tmp_path / "output" + output_dir_path.mkdir() + report_path = tmp_path / "report.json" + + sys.argv = [ + "test_package_python_function", + str(test_data.venv_dir), + "--project", + str(test_data.pyproject.path), + "--output-dir", + str(output_dir_path), + "--report", + str(report_path), + ] + main() + + zip_file = output_dir_path / f"{test_data.pyproject.name.replace('-', '_')}.zip" + report = json.loads(report_path.read_text()) + + assert Path(report["output_file"]) == zip_file.resolve() + assert report["distribution_name"] == test_data.pyproject.name.replace("-", "_") + assert report["nested_zip"] is False + assert report["output_bytes"] == zip_file.stat().st_size + # The single-zip strategy copies the dependencies zip verbatim, so the two figures describe the same bytes. + assert report["compressed_bytes"] == report["output_bytes"] + assert report["uncompressed_bytes"] == _expected_uncompressed_bytes(test_data) + +def test_report_describes_a_nested_zip( + monkeypatch: MonkeyPatch, + test_files: tuple, + tmp_path: Path, +) -> None: + files, files_excluded_from_bundle, loc = test_files + # Compressible bulk, so that the uncompressed size exceeds the limit while the compressed size does not. + test_data = Data.new( + project_name="project-1", + project_files=[*files, File.new("bulky_dependency/bulky.py", "a" * 100_000)], + files_excluded_from_bundle=files_excluded_from_bundle, + ).commit(loc=loc) + + monkeypatch.setattr(Packager, "AWS_LAMBDA_MAX_UNZIP_SIZE", 10_000) + + output_dir_path = tmp_path / "output" + output_dir_path.mkdir() + report_path = tmp_path / "report.json" + + sys.argv = [ + "test_package_python_function", + str(test_data.venv_dir), + "--project", + str(test_data.pyproject.path), + "--output-dir", + str(output_dir_path), + "--report", + str(report_path), + ] + main() + + outer_zip = output_dir_path / f"{test_data.pyproject.name.replace('-', '_')}.zip" + report = json.loads(report_path.read_text()) + + assert Path(report["output_file"]) == outer_zip.resolve() + assert report["nested_zip"] is True + assert report["output_bytes"] == outer_zip.stat().st_size + # compressed_bytes describes the inner dependencies zip, which the outer zip stores alongside the loader. + assert report["compressed_bytes"] < report["output_bytes"] + assert report["uncompressed_bytes"] == _expected_uncompressed_bytes(test_data) + +def test_no_report_is_written_without_the_flag(test_data: Data, tmp_path: Path) -> None: + output_dir_path = tmp_path / "output" + output_dir_path.mkdir() + + sys.argv = [ + "test_package_python_function", + str(test_data.venv_dir), + "--project", + str(test_data.pyproject.path), + "--output-dir", + str(output_dir_path), + ] + main() + + assert [path.name for path in output_dir_path.iterdir()] == [ + f"{test_data.pyproject.name.replace('-', '_')}.zip" + ] + +def test_package_too_large_raises_and_writes_nothing( + monkeypatch: MonkeyPatch, + test_data: Data, + tmp_path: Path, +) -> None: + # A limit this small is exceeded by both figures, which is the only way to reach the failing branch without + # generating hundreds of megabytes of incompressible data. + monkeypatch.setattr(Packager, "AWS_LAMBDA_MAX_UNZIP_SIZE", 10) + + output_dir_path = tmp_path / "output" + output_dir_path.mkdir() + report_path = tmp_path / "report.json" + + sys.argv = [ + "test_package_python_function", + str(test_data.venv_dir), + "--project", + str(test_data.pyproject.path), + "--output-dir", + str(output_dir_path), + "--report", + str(report_path), + ] + + with pytest.raises(PackageTooLargeError) as error: + main() + + assert error.value.uncompressed_bytes > 10 + assert error.value.compressed_bytes > 10 + assert error.value.limit_bytes == 10 + + assert list(output_dir_path.iterdir()) == [] + assert not report_path.exists() + +def test_venv_without_a_python_lib_dir_names_the_path(test_data: Data, tmp_path: Path) -> None: + empty_venv_dir = tmp_path / "empty-venv" + empty_venv_dir.mkdir() + + sys.argv = [ + "test_package_python_function", + str(empty_venv_dir), + "--project", + str(test_data.pyproject.path), + "--output-dir", + str(tmp_path / "output"), + ] + + with pytest.raises(FileNotFoundError, match="lib/python"): + main() + +def test_pyproject_without_a_name_names_what_was_searched(tmp_path: Path) -> None: + pyproject_path = tmp_path / "pyproject.toml" + pyproject_path.write_text('[project]\nversion = "1.2.3"\n') + + with pytest.raises(ValueError, match="project.name, tool.poetry.name"): + PythonProject(pyproject_path).name + +@pytest.mark.parametrize( + "name, expected", + [ + ("My-App", "My_App"), + ("my.app", "my.app"), + ("a b c", "a_b_c"), + # 35 separate runs to replace. re.UNICODE is 32, so passing it as `count` stopped the replacement early. + ("-".join("abcdefghijklmnopqrstuvwxyz0123456789"), "_".join("abcdefghijklmnopqrstuvwxyz0123456789")), + ], + ids=["case_is_preserved", "dots_are_kept", "runs_collapse_to_one_underscore", "every_run_is_replaced"], +) +def test_distribution_name(name: str, expected: str, tmp_path: Path) -> None: + pyproject_path = tmp_path / "pyproject.toml" + pyproject_path.write_text(f'[project]\nname = "{name}"\n') + + assert PythonProject(pyproject_path).distribution_name == expected + +def test_output_filename_preserves_case(test_files: tuple, tmp_path: Path) -> None: + files, files_excluded_from_bundle, loc = test_files + test_data = Data.new( + project_name="My-App", + project_files=files, + files_excluded_from_bundle=files_excluded_from_bundle, + ).commit(loc=loc) + + output_dir_path = tmp_path / "output" + output_dir_path.mkdir() + + sys.argv = [ + "test_package_python_function", + str(test_data.venv_dir), + "--project", + str(test_data.pyproject.path), + "--output-dir", + str(output_dir_path), + ] + main() + + assert (output_dir_path / "My_App.zip").exists() + +@pytest.mark.parametrize( + "src_epoch", + ["notanumber", "420"], + ids=["not_an_integer", "before_1980"], +) +def test_source_date_epoch_is_validated_before_packaging( + monkeypatch: MonkeyPatch, + src_epoch: str, + test_data: Data, + tmp_path: Path, +) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", src_epoch) + + output_dir_path = tmp_path / "output" + output_dir_path.mkdir() + + sys.argv = [ + "test_package_python_function", + str(test_data.venv_dir), + "--project", + str(test_data.pyproject.path), + "--output-dir", + str(output_dir_path), + ] + + with pytest.raises(SourceDateEpochError): + main() + + assert list(output_dir_path.iterdir()) == [] + +def test_output_and_output_dir_are_mutually_exclusive(test_data: Data, tmp_path: Path) -> None: + sys.argv = [ + "test_package_python_function", + str(test_data.venv_dir), + "--project", + str(test_data.pyproject.path), + "--output-dir", + str(tmp_path / "output"), + "--output", + str(tmp_path / "output" / "explicit.zip"), + ] + + with pytest.raises(SystemExit) as error: + main() + + assert error.value.code == 2