Add cluster-init hooks for chefless PBS Pro

.github/workflows/release.yml

@@ -10,16 +10,19 @@ jobs:
   build:
     name: Upload Release Asset
     runs-on: ubuntu-latest
+    container:
+      image: almalinux:8
     permissions:
       contents: write
     steps:
+      - name: Install build prerequisites
+        run: |
+          dnf update -y
+          dnf install -y python3.11 python3.11-pip git tar gzip which findutils
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Build pkg
-        run: |
-          sudo apt update || apt update
-          sudo apt-get install -y python3 python3-pip || apt-get install -y python3 python3-pip
-          ./build.sh
+        run: ./build.sh
       - name: Get the version
         id: get_version
         run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
@@ -41,7 +44,7 @@ jobs:
           generate_release_notes: true
           files: |
             blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.VERSION }}.tar.gz
-            blobs/cyclecloud_api-8.3.1-py2.py3-none-any.whl
+            blobs/cyclecloud_api-8.10.0-py2.py3-none-any.whl
             blobs/hwloc-libs-1.11.9-3.el8.x86_64.rpm
             blobs/openpbs-client-20.0.1-0.x86_64.rpm
             blobs/openpbs-client-22.05.11-0.x86_64.rpm

.gitignore

@@ -22,3 +22,4 @@ dist/**
 .env
 libs/**
 venv/**
+.build.log

README.md

@@ -37,8 +37,6 @@ cd cyclecloud-pbspro
 
 # If you have jetpack available, you may use the following:
 # ./generate_autoscale_json.sh --install-dir /opt/cycle/pbspro \
-#                              --username $(jetpack config cyclecloud.config.username) \
-#                              --password $(jetpack config cyclecloud.config.password) \
 #                              --url $(jetpack config cyclecloud.config.web_server) \
 #                              --cluster-name $(jetpack config cyclecloud.cluster.name)
 

build.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
+set -e
 
 # create a new venv if it does not exist, or is older than 7 days
-if [ ! $(find . -path ./venv/created -mtime -7) ]; then
+if [ -z "$(find . -path ./venv/created -mtime -7 -print -quit)" ] ||
+    ! venv/bin/python -c 'import sys; raise SystemExit(sys.version_info < (3, 11))' 2>/dev/null; then
     rm -rf venv
     python3 -m venv venv
     source venv/bin/activate
-    pip install setuptools
+    python3 -m pip install setuptools
     touch venv/created
 else
     source venv/bin/activate
 fi
 
-python package.py 
-
+./docker-package.sh

docker-package.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+echo see .build.log for more information
+log_file=".build.log"
+check_dirty_changes() {
+  if [ -n "$(git status --porcelain)" ]; then
+    echo "Error: There are uncommitted changes in the current branch. Please commit or stash them before running this script."
+    exit 1
+  fi
+}
+
+# Call the function to check for dirty changes
+# check_dirty_changes
+
+print_branch_and_last_commit() {
+  branch=$(git rev-parse --abbrev-ref HEAD)
+  last_commit=$(git log -1 --pretty=format:"%h - %s (%ci)")
+
+  printf "%-20s: %s\n" "Current branch" "$branch"
+  printf "%-20s: %s\n" "Last commit" "$last_commit"
+}
+
+delete_existing_blobs() {
+  printf "%-20s: %s\n" "Deleting existing Blob Files" ""
+  while IFS= read -r file; do
+    if [ -f "blobs/$file" ]; then
+      printf "%-20s: %s\n" "" "$file"
+      rm -f "blobs/$file" || exit 1
+    fi
+  done < <(awk -F' *= *' '/^\[blobs\]/ {found=1} found && /^Files/ {gsub(/, */, "\n", $2); print $2; exit}' project.ini)
+}
+
+check_blobs_files_exist() {
+  local version="$1"
+  local missing_files=0
+
+  printf "%-20s: %s\n" "Blob Files" ""
+  while IFS= read -r file; do
+    printf "%-20s: %s\n" "" "$file"
+    if [ ! -f "blobs/$file" ]; then
+      echo "Error: File blobs/$file does not exist."
+      missing_files=1
+    fi
+  done < <(awk -F' *= *' '/^\[blobs\]/ {found=1} found && /^Files/ {gsub(/, */, "\n", $2); print $2; exit}' project.ini)
+
+  if [ $missing_files -eq 1 ]; then
+    echo "One or more required files are missing in the blobs directory."
+    exit 1
+
+  fi
+}
+
+get_version_from_project_ini() {
+  version=$(awk -F' *= *' '/^\[project\]/ {found=1} found && /^version/ {print $2; exit}' project.ini)
+  printf "%-20s: %s\n" "Project Version" "$version"
+  check_blobs_files_exist "$version"
+}
+
+local_azpbs=/source/
+if [ "$1" != "" ]; then
+  scalelib=$(realpath $1)
+  local_scalelib=/source/cyclecloud-scalelib
+  extra_args="-v ${scalelib}:${local_scalelib}"
+fi
+
+if command -v docker; then
+  runtime=docker
+  runtime_args=
+elif command -v podman; then
+  runtime=podman
+  runtime_args="--privileged"
+else
+  echo "`docker` or `podman` binary not found. Install docker or podman to build RPMs with this script"
+  exit 1
+fi
+
+{
+  delete_existing_blobs
+  # allows caching
+  $runtime build -t azpbs_build:latest -f util/Dockerfile .
+  $runtime run --user "$(id -u):$(id -g)" -e HOME=/tmp -v "$(pwd):${local_azpbs}" $runtime_args $extra_args -ti azpbs_build:latest /bin/bash ${local_azpbs}/util/build.sh $local_scalelib
+} &> $log_file
+
+# Call the function to print the branch and the last commit
+print_branch_and_last_commit
+get_version_from_project_ini

docker-rpmbuild.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if command -v docker; then
-  docker run -v $(pwd)/specs/default/cluster-init/files:/source -v $(pwd)/blobs:/root/rpmbuild/RPMS/x86_64 -ti centos:7 /bin/bash -e /source/00-build-pbspro.sh
+  docker run -v $(pwd)/specs/default/cluster-init/files:/source -v $(pwd)/blobs:/root/rpmbuild/RPMS/x86_64 -ti  almalinux:8 /bin/bash -e /source/00-build-pbspro.sh
 else
   echo "`docker` binary not found. Install docker to build RPMs with this script"
 fi

generate_autoscale_json.sh

@@ -28,7 +28,7 @@ CLUSTER_NAME=
 IGNORE_QUEUES_ARG=
 
 function usage() {
-    echo Usage: $0 --username username --password password --url https://fqdn:port --cluster-name cluster_name [--install-dir /opt/cycle/pbspro]
+    echo Usage: $0 --url https://fqdn:port --cluster-name cluster_name [--username username] [--password password] [--install-dir /opt/cycle/pbspro]
     exit 2
 }
 
@@ -73,7 +73,9 @@ done
 if [ "$1" == "-h" ]; then usage; fi
 if [ "$1" == "-help" ]; then usage; fi
 
+if [ "$USERNAME" == "" ]; then USERNAME=$(jetpack config cyclecloud.config.username); fi
 if [ "$USERNAME" == "" ]; then usage; fi
+if [ "$PASSWORD" == "" ]; then PASSWORD=$(jetpack config cyclecloud.config.password); fi
 if [ "$PASSWORD" == "" ]; then usage; fi
 if [ "$URL" == "" ]; then usage; fi
 if [ "$CLUSTER_NAME" == "" ]; then usage; fi

install.sh

@@ -56,35 +56,45 @@ echo VENV=$VENV
 # remove jetpack's python3 from the path
 export PATH=$(echo "$PATH" | sed -e 's/\/opt\/cycle\/jetpack\/system\/embedded\/bin://g' | sed -e 's/:\/opt\/cycle\/jetpack\/system\/embedded\/bin//g')
 set +e
-which python3 > /dev/null;
+which python3.11 > /dev/null;
 if [ $? != 0 ]; then
     if [ $INSTALL_PYTHON3 == 1 ]; then
-        yum install -y -q python3 || exit 1
+        yum install -y -q python3.11 || exit 1
     else
-        echo Please install python3 >&2;
+        echo Please install python3.11 >&2;
+        exit 1
+    fi
+fi
+
+python3.11 -m pip --version > /dev/null 2>&1
+if [ $? != 0 ]; then
+    if [ $INSTALL_PYTHON3 == 1 ]; then
+        yum install -y -q python3.11-pip || exit 1
+    else
+        echo Please install pip for python3.11 >&2;
         exit 1
     fi
 fi
 set -e
 
 if [ $INSTALL_VIRTUALENV == 1 ]; then
-    python3 -m pip install -q virtualenv
+    python3.11 -m pip install -q virtualenv
 fi
 
 set +e
-python3 -m virtualenv --version 2>&1 > /dev/null
+python3.11 -m virtualenv --version 2>&1 > /dev/null
 
 if [ $? != 0 ]; then
     if [ $INSTALL_VIRTUALENV ]; then
-        python3 -m pip install -q virtualenv || exit 1
+        python3.11 -m pip install -q virtualenv || exit 1
     else
         echo Please install virtualenv for python3 >&2
         exit 1
     fi
 fi
 set -e
 
-python3 -m virtualenv $VENV
+python3.11 -m virtualenv $VENV
 source "${VENV}/bin/activate"
 # not sure why but pip gets confused installing frozendict locally
 # if you don't install it first. It has no dependencies so this is safe.

package.py

@@ -11,8 +11,8 @@
 from typing import Dict, List, Optional
 from util import download_release_files
 
-SCALELIB_VERSION = "1.0.11"
-CYCLECLOUD_API_VERSION = "8.3.1"
+SCALELIB_VERSION = "1.0.12"
+CYCLECLOUD_API_VERSION = "8.10.0"
 
 
 def build_sdist() -> str:
@@ -39,7 +39,7 @@ def get_cycle_packages(args: Namespace) -> List[str]:
 
     scalelib_url = f"https://github.com/Azure/cyclecloud-scalelib/archive/refs/tags/{SCALELIB_VERSION}.tar.gz"
 
-    cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/{cyclecloud_api_file}"
+    cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2.0.26/{cyclecloud_api_file}"
     to_download = {
         scalelib_file: (args.scalelib, scalelib_url),
         cyclecloud_api_file: (args.cyclecloud_api, cyclecloud_api_url),
@@ -119,7 +119,7 @@ def _add(name: str, path: Optional[str] = None, mode: Optional[int] = None) -> N
         _add("packages/" + dep, dep_path)
         packages.append(dep_path)
 
-    check_call(["pip", "download"] + packages, cwd=build_dir)
+    check_call([sys.executable, "-m", "pip", "download"] + packages, cwd=build_dir)
 
     print("Using build dir", build_dir)
     by_package: Dict[str, List[str]] = {}

package.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+cd $(dirname $0)/
+
+
+if [ ! -e libs ]; then
+    mkdir libs
+fi
+
+LOCAL_SCALELIB=$1
+
+rm -f dist/*
+
+if [ "$LOCAL_SCALELIB" == "" ]; then
+    # we are using released versions of scalelib
+    python3.11 package.py
+else
+    pushd $LOCAL_SCALELIB
+    rm -f dist/*.gz
+    # python3 setup.py swagger
+    python3.11 setup.py sdist
+    popd
+    # swagger=`ls $LOCAL_SCALELIB/dist/swagger*.gz`
+    scalelib=`ls $LOCAL_SCALELIB/dist/cyclecloud_scalelib*.gz`
+    # python3 package.py --scalelib $scalelib --swagger $swagger
+    python3.11 package.py --scalelib $scalelib
+fi

project.ini

@@ -6,7 +6,7 @@ version = 2.0.26
 autoupgrade = true
 
 [blobs]
-Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, cyclecloud_api-8.3.1-py2.py3-none-any.whl, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm
+Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, cyclecloud_api-8.10.0-py2.py3-none-any.whl, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm
 
 [spec server]
 

specs/default/cluster-init/activate-hook.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1
+ROLE=$(jetpack config pbspro.role "") || fail
+
+case "$ROLE" in
+    execute)      systemctl start pbs || exit 1 ;;
+    server|login) ;;
+    *)            fail "Unknown pbspro.role '$ROLE'" ;;
+esac

specs/default/cluster-init/configure-hook.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1
+
+ROLE=$(jetpack config pbspro.role "") || fail
+
+case "$ROLE" in
+    server)  bash "${CYCLECLOUD_PROJECT_PATH}/default/files/skel.sh" || fail ;;
+    login|execute) ;;
+    *)            fail "Unknown pbspro.role '$ROLE'" ;;
+esac

specs/default/cluster-init/scripts/hwlocs-install.sh → specs/default/cluster-init/files/hwlocs-install.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1
+source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail
+
 PLATFORM_VERSION=$(jetpack props get os.version) || fail
 
 if [[ "${PLATFORM_VERSION%%.*}" -ge 8 ]]; then

specs/default/cluster-init/files/utils.sh

@@ -66,4 +66,4 @@ function get_server_hostname() {
     else
         echo "$server_hostname"
     fi
-}
+}

specs/default/cluster-init/install-hook.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1
+
+ROLE=$(jetpack config pbspro.role "") || fail
+
+case "$ROLE" in
+    server)
+        PACKAGE_TYPE="server"
+        ROLE_SCRIPT="server-install.sh"
+        ;;
+    login)
+        PACKAGE_TYPE="client"
+        ROLE_SCRIPT="login-install.sh"
+        ;;
+    execute)
+        PACKAGE_TYPE="execution"
+        ROLE_SCRIPT="execute-install.sh"
+        ;;
+    *)       fail "Unknown pbspro.role '$ROLE'" ;;
+esac
+
+bash "${CYCLECLOUD_PROJECT_PATH}/default/files/hwlocs-install.sh" || fail
+
+PACKAGE_NAME=$(get_package_name "$PACKAGE_TYPE") || fail
+jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail
+yum install -y -q "/tmp/$PACKAGE_NAME" || fail
+
+bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/${ROLE_SCRIPT}" || fail

specs/execute/cluster-init/stages/install.sh → specs/default/cluster-init/roles/execute-install.sh

@@ -1,21 +1,15 @@
 #!/bin/bash
 
 source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1
-source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail
-
-"${CYCLECLOUD_PROJECT_PATH}/default/scripts/hwlocs-install.sh" || fail
 
 EXECUTE_HOSTNAME=$(jetpack config hostname) || fail
-PACKAGE_NAME=$(get_package_name "execution") || fail
 SERVER_HOSTNAME=$(get_server_hostname) || fail
 
+
 # Forces execute node's hostname to be updated (scalelib is blocked until the hostname is correct)
 # TODO: this installation status should be done by jetpack before cluster-inits are run
 "${CYCLECLOUD_HOME}/system/embedded/bin/python" -c "import jetpack.converge as jc; jc._send_installation_status('warning')"
 
-jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail
-yum install -y -q "/tmp/$PACKAGE_NAME" || fail
-
 if [[ -n "$SERVER_HOSTNAME" ]]; then
     echo "$SERVER_HOSTNAME" > /var/spool/pbs/server_name
     chmod 0644 /var/spool/pbs/server_name || fail