From 0d4ddf989e24c1e90fc35cf05df78c349a88a3ae Mon Sep 17 00:00:00 2001 From: Ryan Hamel Date: Thu, 17 Sep 2026 12:03:51 -0400 Subject: [PATCH 1/2] Use scalelib 1.0.12 and included api whl --- .github/workflows/release.yml | 2 +- package.py | 46 ++++++++++++++++--- project.ini | 2 +- test_packaging/test_package.py | 84 ++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 9 deletions(-) create mode 100644 test_packaging/test_package.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b803ab..01910cd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,7 +41,7 @@ jobs: generate_release_notes: true files: | blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.VERSION }}.tar.gz - blobs/cyclecloud_api-8.3.1-py2.py3-none-any.whl + blobs/cyclecloud_api-*.whl blobs/hwloc-libs-1.11.9-3.el8.x86_64.rpm blobs/openpbs-client-20.0.1-0.x86_64.rpm blobs/openpbs-client-22.05.11-0.x86_64.rpm diff --git a/package.py b/package.py index d917b8f..3276a14 100644 --- a/package.py +++ b/package.py @@ -11,8 +11,42 @@ from typing import Dict, List, Optional from util import download_release_files -SCALELIB_VERSION = "1.0.11" -CYCLECLOUD_API_VERSION = "8.3.1" +SCALELIB_VERSION = "1.0.12" + + +def get_cyclecloud_api(override: Optional[str]) -> str: + if override: + if not os.path.isfile(override): + raise FileNotFoundError(override) + fname = os.path.basename(override) + dest = os.path.abspath(os.path.join("blobs", fname)) + if os.path.abspath(override) != dest: + shutil.copyfile(override, dest) + return fname + + archive_name = f"cyclecloud-scalelib-pkg-{SCALELIB_VERSION}.tar.gz" + url = f"https://github.com/Azure/cyclecloud-scalelib/releases/download/{SCALELIB_VERSION}/{archive_name}" + with tempfile.TemporaryDirectory() as download_dir: + archive_path = os.path.join(download_dir, archive_name) + check_call(["curl", "-L", "-s", "-f", "-o", archive_path, url]) + with tarfile.open(archive_path, "r:gz") as archive: + wheels = { + member.name: member for member in archive.getmembers() + if member.isfile() + and os.path.dirname(member.name) == "cyclecloud-scalelib/packages" + and os.path.basename(member.name).startswith("cyclecloud_api-") + and member.name.endswith(".whl") + } + if len(wheels) != 1: + raise RuntimeError( + f"Expected one CycleCloud API wheel in {archive_name}, found {len(wheels)}" + ) + wheel = next(iter(wheels.values())) + fname = os.path.basename(wheel.name) + with archive.extractfile(wheel) as source: + with open(os.path.join("blobs", fname), "wb") as dest: + shutil.copyfileobj(source, dest) + return fname def build_sdist() -> str: @@ -35,14 +69,11 @@ def get_cycle_packages(args: Namespace) -> List[str]: ret = [build_sdist()] scalelib_file = "cyclecloud-scalelib-{}.tar.gz".format(SCALELIB_VERSION) - cyclecloud_api_file = f"cyclecloud_api-{CYCLECLOUD_API_VERSION}-py2.py3-none-any.whl" scalelib_url = f"https://github.com/Azure/cyclecloud-scalelib/archive/refs/tags/{SCALELIB_VERSION}.tar.gz" - cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/{cyclecloud_api_file}" to_download = { scalelib_file: (args.scalelib, scalelib_url), - cyclecloud_api_file: (args.cyclecloud_api, cyclecloud_api_url), } for pkg_file in to_download: @@ -53,17 +84,18 @@ def get_cycle_packages(args: Namespace) -> List[str]: sys.exit(1) fname = os.path.basename(arg_override) orig = os.path.abspath(arg_override) - dest = os.path.abspath(os.path.join("blobs" if pkg_file == cyclecloud_api_file else "libs", fname)) + dest = os.path.abspath(os.path.join("libs", fname)) if orig != dest: shutil.copyfile(orig, dest) ret.append(fname) else: - dest = os.path.join("blobs" if pkg_file == cyclecloud_api_file else "libs", pkg_file) + dest = os.path.join("libs", pkg_file) check_call(["curl", "-L", "-s", "-f", "-z", dest, "-o", dest, url]) ret.append(pkg_file) print("Downloaded", pkg_file, "to", dest) + ret.append(get_cyclecloud_api(args.cyclecloud_api)) return ret def execute() -> None: diff --git a/project.ini b/project.ini index 35e0a24..28325c6 100644 --- a/project.ini +++ b/project.ini @@ -6,7 +6,7 @@ version = 2.0.26 autoupgrade = true [blobs] -Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, cyclecloud_api-8.3.1-py2.py3-none-any.whl, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm +Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm [spec server] diff --git a/test_packaging/test_package.py b/test_packaging/test_package.py new file mode 100644 index 0000000..1e4b9c4 --- /dev/null +++ b/test_packaging/test_package.py @@ -0,0 +1,84 @@ +import io +import os +from pathlib import Path +import tarfile +import tempfile +import unittest +from argparse import Namespace +from unittest.mock import patch + +import package +import util + + +class CycleCloudApiTests(unittest.TestCase): + def setUp(self): + self.original_cwd = os.getcwd() + self.temp_dir = tempfile.TemporaryDirectory() + os.chdir(self.temp_dir.name) + self.addCleanup(self.temp_dir.cleanup) + self.addCleanup(os.chdir, self.original_cwd) + Path("blobs").mkdir() + Path("libs").mkdir() + + def download_archive(self, command): + self.assertEqual(command[-1], + "https://github.com/Azure/cyclecloud-scalelib/releases/download/" + "1.2.3/cyclecloud-scalelib-pkg-1.2.3.tar.gz") + with tarfile.open(command[command.index("-o") + 1], "w:gz") as archive: + for name in self.members: + member = tarfile.TarInfo("cyclecloud-scalelib/packages/" + name) + member.size = len(b"wheel contents") + archive.addfile(member, io.BytesIO(b"wheel contents")) + + def test_uses_api_wheel_from_selected_scalelib_release(self): + wheel = "cyclecloud_api-9.2.1-py2.py3-none-any.whl" + self.members = ["cyclecloud-scalelib-1.2.3.tar.gz", wheel] + Path("local-scalelib.tar.gz").touch() + with patch.object(package, "SCALELIB_VERSION", "1.2.3"), \ + patch.object(package, "build_sdist", return_value="pbspro.tar.gz"), \ + patch.object(package, "check_call", side_effect=self.download_archive): + packages = package.get_cycle_packages( + Namespace(scalelib="local-scalelib.tar.gz", cyclecloud_api=None) + ) + self.assertEqual(packages, ["pbspro.tar.gz", "local-scalelib.tar.gz", wheel]) + self.assertEqual(Path("blobs", wheel).read_bytes(), b"wheel contents") + + def test_duplicate_archive_entries_for_same_wheel_are_allowed(self): + wheel = "cyclecloud_api-9.2.1-py2.py3-none-any.whl" + self.members = [wheel, wheel] + with patch.object(package, "SCALELIB_VERSION", "1.2.3"), \ + patch.object(package, "check_call", side_effect=self.download_archive): + self.assertEqual(package.get_cyclecloud_api(None), wheel) + self.assertEqual(Path("blobs", wheel).read_bytes(), b"wheel contents") + + def test_local_override_does_not_download(self): + wheel = "cyclecloud_api-9.2.2-py2.py3-none-any.whl" + Path(wheel).write_bytes(b"local wheel") + with patch.object(package, "check_call") as download: + self.assertEqual(package.get_cyclecloud_api(wheel), wheel) + self.assertEqual(package.get_cyclecloud_api(str(Path("blobs", wheel))), wheel) + download.assert_not_called() + self.assertEqual(Path("blobs", wheel).read_bytes(), b"local wheel") + + def test_missing_or_ambiguous_api_wheel_fails(self): + for members in ([], ["cyclecloud_api-1.whl", "cyclecloud_api-2.whl"]): + with self.subTest(members=members): + self.members = members + with patch.object(package, "SCALELIB_VERSION", "1.2.3"), \ + patch.object(package, "check_call", side_effect=self.download_archive): + with self.assertRaisesRegex(RuntimeError, "Expected one CycleCloud API wheel"): + package.get_cyclecloud_api(None) + self.assertEqual(list(Path("blobs").iterdir()), []) + + def test_legacy_release_downloads_do_not_include_api_wheel(self): + os.chdir(Path(package.__file__).parent) + with patch.object(util, "run") as download: + util.download_release_files() + self.assertTrue(download.called) + for call in download.call_args_list: + self.assertFalse(call.args[0][-1].endswith(".whl")) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 157c63a8eb4127cc14201ff03e59f4045af3f8ed Mon Sep 17 00:00:00 2001 From: Ryan Hamel Date: Thu, 17 Sep 2026 12:17:14 -0400 Subject: [PATCH 2/2] Fix run_tests.sh given the packaging changes --- run_tests.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 616d7f9..e6a80f0 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -7,9 +7,18 @@ cd "$PROJECT_ROOT" "${PYTHON:-python3}" -m venv --clear "$PROJECT_ROOT/.testenv" TEST_PYTHON="$PROJECT_ROOT/.testenv/bin/python" SCALELIB_VERSION=$("$TEST_PYTHON" -c 'from package import SCALELIB_VERSION; print(SCALELIB_VERSION)') -API_VERSION=$("$TEST_PYTHON" -c 'from package import CYCLECLOUD_API_VERSION; print(CYCLECLOUD_API_VERSION)') SCALELIB=${CYCLECLOUD_SCALELIB:-"https://github.com/Azure/cyclecloud-scalelib/archive/refs/tags/$SCALELIB_VERSION.tar.gz"} -API_WHEEL=${CYCLECLOUD_API:-"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/cyclecloud_api-$API_VERSION-py2.py3-none-any.whl"} +API_WHEEL=${CYCLECLOUD_API:-} +if [[ -z "$API_WHEEL" ]]; then + API_WHEEL=$("$TEST_PYTHON" - <<'PY' +import os +from package import get_cyclecloud_api + +os.makedirs("blobs", exist_ok=True) +print(os.path.abspath(os.path.join("blobs", get_cyclecloud_api(None)))) +PY + ) +fi "$TEST_PYTHON" -m pip install --upgrade pip 'setuptools<72' wheel "$TEST_PYTHON" -m pip install --no-build-isolation \