Use scalelib 1.0.12 and include cylecloud_api asset

.github/workflows/release.yml

@@ -41,7 +41,7 @@ jobs:
           generate_release_notes: true
           files: |
             blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.VERSION }}.tar.gz
-            blobs/cyclecloud_api-8.3.1-py2.py3-none-any.whl
+            blobs/cyclecloud_api-*.whl
             blobs/hwloc-libs-1.11.9-3.el8.x86_64.rpm
             blobs/openpbs-client-20.0.1-0.x86_64.rpm
             blobs/openpbs-client-22.05.11-0.x86_64.rpm

package.py

@@ -11,8 +11,42 @@
 from typing import Dict, List, Optional
 from util import download_release_files
 
-SCALELIB_VERSION = "1.0.11"
-CYCLECLOUD_API_VERSION = "8.3.1"
+SCALELIB_VERSION = "1.0.12"
+
+
+def get_cyclecloud_api(override: Optional[str]) -> str:
+    if override:
+        if not os.path.isfile(override):
+            raise FileNotFoundError(override)
+        fname = os.path.basename(override)
+        dest = os.path.abspath(os.path.join("blobs", fname))
+        if os.path.abspath(override) != dest:
+            shutil.copyfile(override, dest)
+        return fname
+
+    archive_name = f"cyclecloud-scalelib-pkg-{SCALELIB_VERSION}.tar.gz"
+    url = f"https://github.com/Azure/cyclecloud-scalelib/releases/download/{SCALELIB_VERSION}/{archive_name}"
+    with tempfile.TemporaryDirectory() as download_dir:
+        archive_path = os.path.join(download_dir, archive_name)
+        check_call(["curl", "-L", "-s", "-f", "-o", archive_path, url])
+        with tarfile.open(archive_path, "r:gz") as archive:
+            wheels = {
+                member.name: member for member in archive.getmembers()
+                if member.isfile()
+                and os.path.dirname(member.name) == "cyclecloud-scalelib/packages"
+                and os.path.basename(member.name).startswith("cyclecloud_api-")
+                and member.name.endswith(".whl")
+            }
+            if len(wheels) != 1:
+                raise RuntimeError(
+                    f"Expected one CycleCloud API wheel in {archive_name}, found {len(wheels)}"
+                )
+            wheel = next(iter(wheels.values()))
+            fname = os.path.basename(wheel.name)
+            with archive.extractfile(wheel) as source:
+                with open(os.path.join("blobs", fname), "wb") as dest:
+                    shutil.copyfileobj(source, dest)
+    return fname
 
 
 def build_sdist() -> str:
@@ -35,14 +69,11 @@ def get_cycle_packages(args: Namespace) -> List[str]:
     ret = [build_sdist()]
 
     scalelib_file = "cyclecloud-scalelib-{}.tar.gz".format(SCALELIB_VERSION)
-    cyclecloud_api_file = f"cyclecloud_api-{CYCLECLOUD_API_VERSION}-py2.py3-none-any.whl"
 
     scalelib_url = f"https://github.com/Azure/cyclecloud-scalelib/archive/refs/tags/{SCALELIB_VERSION}.tar.gz"
 
-    cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/{cyclecloud_api_file}"
     to_download = {
         scalelib_file: (args.scalelib, scalelib_url),
-        cyclecloud_api_file: (args.cyclecloud_api, cyclecloud_api_url),
     }
 
     for pkg_file in to_download:
@@ -53,17 +84,18 @@ def get_cycle_packages(args: Namespace) -> List[str]:
                 sys.exit(1)
             fname = os.path.basename(arg_override)
             orig = os.path.abspath(arg_override)
-            dest = os.path.abspath(os.path.join("blobs" if pkg_file == cyclecloud_api_file else "libs", fname))
+            dest = os.path.abspath(os.path.join("libs", fname))
             if orig != dest:
                 shutil.copyfile(orig, dest)
             ret.append(fname)
         else:
-            dest = os.path.join("blobs" if pkg_file == cyclecloud_api_file else "libs", pkg_file)
+            dest = os.path.join("libs", pkg_file)
             check_call(["curl", "-L", "-s", "-f", "-z", dest, "-o", dest, url])
 
             ret.append(pkg_file)
             print("Downloaded", pkg_file, "to", dest)
 
+    ret.append(get_cyclecloud_api(args.cyclecloud_api))
     return ret
 
 def execute() -> None:

project.ini

@@ -6,7 +6,7 @@ version = 2.0.26
 autoupgrade = true
 
 [blobs]
-Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, cyclecloud_api-8.3.1-py2.py3-none-any.whl, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm
+Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm
 
 [spec server]
 

run_tests.sh

@@ -7,9 +7,18 @@ cd "$PROJECT_ROOT"
 "${PYTHON:-python3}" -m venv --clear "$PROJECT_ROOT/.testenv"
 TEST_PYTHON="$PROJECT_ROOT/.testenv/bin/python"
 SCALELIB_VERSION=$("$TEST_PYTHON" -c 'from package import SCALELIB_VERSION; print(SCALELIB_VERSION)')
-API_VERSION=$("$TEST_PYTHON" -c 'from package import CYCLECLOUD_API_VERSION; print(CYCLECLOUD_API_VERSION)')
 SCALELIB=${CYCLECLOUD_SCALELIB:-"https://github.com/Azure/cyclecloud-scalelib/archive/refs/tags/$SCALELIB_VERSION.tar.gz"}
-API_WHEEL=${CYCLECLOUD_API:-"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/cyclecloud_api-$API_VERSION-py2.py3-none-any.whl"}
+API_WHEEL=${CYCLECLOUD_API:-}
+if [[ -z "$API_WHEEL" ]]; then
+    API_WHEEL=$("$TEST_PYTHON" - <<'PY'
+import os
+from package import get_cyclecloud_api
+
+os.makedirs("blobs", exist_ok=True)
+print(os.path.abspath(os.path.join("blobs", get_cyclecloud_api(None))))
+PY
+    )
+fi
 
 "$TEST_PYTHON" -m pip install --upgrade pip 'setuptools<72' wheel
 "$TEST_PYTHON" -m pip install --no-build-isolation \

test_packaging/test_package.py

@@ -0,0 +1,84 @@
+import io
+import os
+from pathlib import Path
+import tarfile
+import tempfile
+import unittest
+from argparse import Namespace
+from unittest.mock import patch
+
+import package
+import util
+
+
+class CycleCloudApiTests(unittest.TestCase):
+    def setUp(self):
+        self.original_cwd = os.getcwd()
+        self.temp_dir = tempfile.TemporaryDirectory()
+        os.chdir(self.temp_dir.name)
+        self.addCleanup(self.temp_dir.cleanup)
+        self.addCleanup(os.chdir, self.original_cwd)
+        Path("blobs").mkdir()
+        Path("libs").mkdir()
+
+    def download_archive(self, command):
+        self.assertEqual(command[-1],
+                         "https://github.com/Azure/cyclecloud-scalelib/releases/download/"
+                         "1.2.3/cyclecloud-scalelib-pkg-1.2.3.tar.gz")
+        with tarfile.open(command[command.index("-o") + 1], "w:gz") as archive:
+            for name in self.members:
+                member = tarfile.TarInfo("cyclecloud-scalelib/packages/" + name)
+                member.size = len(b"wheel contents")
+                archive.addfile(member, io.BytesIO(b"wheel contents"))
+
+    def test_uses_api_wheel_from_selected_scalelib_release(self):
+        wheel = "cyclecloud_api-9.2.1-py2.py3-none-any.whl"
+        self.members = ["cyclecloud-scalelib-1.2.3.tar.gz", wheel]
+        Path("local-scalelib.tar.gz").touch()
+        with patch.object(package, "SCALELIB_VERSION", "1.2.3"), \
+                patch.object(package, "build_sdist", return_value="pbspro.tar.gz"), \
+                patch.object(package, "check_call", side_effect=self.download_archive):
+            packages = package.get_cycle_packages(
+                Namespace(scalelib="local-scalelib.tar.gz", cyclecloud_api=None)
+            )
+        self.assertEqual(packages, ["pbspro.tar.gz", "local-scalelib.tar.gz", wheel])
+        self.assertEqual(Path("blobs", wheel).read_bytes(), b"wheel contents")
+
+    def test_duplicate_archive_entries_for_same_wheel_are_allowed(self):
+        wheel = "cyclecloud_api-9.2.1-py2.py3-none-any.whl"
+        self.members = [wheel, wheel]
+        with patch.object(package, "SCALELIB_VERSION", "1.2.3"), \
+                patch.object(package, "check_call", side_effect=self.download_archive):
+            self.assertEqual(package.get_cyclecloud_api(None), wheel)
+        self.assertEqual(Path("blobs", wheel).read_bytes(), b"wheel contents")
+
+    def test_local_override_does_not_download(self):
+        wheel = "cyclecloud_api-9.2.2-py2.py3-none-any.whl"
+        Path(wheel).write_bytes(b"local wheel")
+        with patch.object(package, "check_call") as download:
+            self.assertEqual(package.get_cyclecloud_api(wheel), wheel)
+            self.assertEqual(package.get_cyclecloud_api(str(Path("blobs", wheel))), wheel)
+        download.assert_not_called()
+        self.assertEqual(Path("blobs", wheel).read_bytes(), b"local wheel")
+
+    def test_missing_or_ambiguous_api_wheel_fails(self):
+        for members in ([], ["cyclecloud_api-1.whl", "cyclecloud_api-2.whl"]):
+            with self.subTest(members=members):
+                self.members = members
+                with patch.object(package, "SCALELIB_VERSION", "1.2.3"), \
+                        patch.object(package, "check_call", side_effect=self.download_archive):
+                    with self.assertRaisesRegex(RuntimeError, "Expected one CycleCloud API wheel"):
+                        package.get_cyclecloud_api(None)
+        self.assertEqual(list(Path("blobs").iterdir()), [])
+
+    def test_legacy_release_downloads_do_not_include_api_wheel(self):
+        os.chdir(Path(package.__file__).parent)
+        with patch.object(util, "run") as download:
+            util.download_release_files()
+        self.assertTrue(download.called)
+        for call in download.call_args_list:
+            self.assertFalse(call.args[0][-1].endswith(".whl"))
+
+
+if __name__ == "__main__":
+    unittest.main()