From 8b807593be756e969911b58483fb7dbf9a56f1d8 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Fri, 14 Aug 2026 18:51:26 +0000 Subject: [PATCH 01/18] Add initial hook setup --- specs/default/cluster-init/activate-hook.sh | 10 +++ specs/default/cluster-init/configure-hook.sh | 11 +++ .../{scripts => files}/hwlocs-install.sh | 3 + .../cluster-init/files}/skel.sh | 0 specs/default/cluster-init/install-hook.sh | 15 ++++ .../cluster-init/roles/execute-install.sh | 67 +++++++++++++++ .../cluster-init/roles/login-install.sh | 17 ++++ .../cluster-init/roles/server-install.sh | 83 +++++++++++++++++++ 8 files changed, 206 insertions(+) create mode 100644 specs/default/cluster-init/activate-hook.sh create mode 100644 specs/default/cluster-init/configure-hook.sh rename specs/default/cluster-init/{scripts => files}/hwlocs-install.sh (65%) rename specs/{server/cluster-init/scripts => default/cluster-init/files}/skel.sh (100%) create mode 100644 specs/default/cluster-init/install-hook.sh create mode 100755 specs/default/cluster-init/roles/execute-install.sh create mode 100755 specs/default/cluster-init/roles/login-install.sh create mode 100644 specs/default/cluster-init/roles/server-install.sh diff --git a/specs/default/cluster-init/activate-hook.sh b/specs/default/cluster-init/activate-hook.sh new file mode 100644 index 0000000..6dfd1c6 --- /dev/null +++ b/specs/default/cluster-init/activate-hook.sh @@ -0,0 +1,10 @@ +#!/bin/bash +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 +echo "im running activate hook" +ROLE=$(jetpack config pbspro.role "") || fail + +case "$ROLE" in + execute) systemctl start pbs || exit 1 ;; + server|login) ;; + *) fail "Unknown pbspro.role '$ROLE'" ;; +esac diff --git a/specs/default/cluster-init/configure-hook.sh b/specs/default/cluster-init/configure-hook.sh new file mode 100644 index 0000000..0bf135e --- /dev/null +++ b/specs/default/cluster-init/configure-hook.sh @@ -0,0 +1,11 @@ +#!/bin/bash +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 +echo "im running configure hook" + +ROLE=$(jetpack config pbspro.role "") || fail + +case "$ROLE" in + server) bash "${CYCLECLOUD_PROJECT_PATH}/default/files/skel.sh" || fail ;; + login|execute) ;; + *) fail "Unknown pbspro.role '$ROLE'" ;; +esac \ No newline at end of file diff --git a/specs/default/cluster-init/scripts/hwlocs-install.sh b/specs/default/cluster-init/files/hwlocs-install.sh similarity index 65% rename from specs/default/cluster-init/scripts/hwlocs-install.sh rename to specs/default/cluster-init/files/hwlocs-install.sh index 35ec467..d98a2f9 100755 --- a/specs/default/cluster-init/scripts/hwlocs-install.sh +++ b/specs/default/cluster-init/files/hwlocs-install.sh @@ -1,5 +1,8 @@ #!/bin/bash +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 +source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail + PLATFORM_VERSION=$(jetpack props get os.version) || fail if [[ "${PLATFORM_VERSION%%.*}" -ge 8 ]]; then diff --git a/specs/server/cluster-init/scripts/skel.sh b/specs/default/cluster-init/files/skel.sh similarity index 100% rename from specs/server/cluster-init/scripts/skel.sh rename to specs/default/cluster-init/files/skel.sh diff --git a/specs/default/cluster-init/install-hook.sh b/specs/default/cluster-init/install-hook.sh new file mode 100644 index 0000000..7df6e49 --- /dev/null +++ b/specs/default/cluster-init/install-hook.sh @@ -0,0 +1,15 @@ +#!/bin/bash +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 + +bash "${CYCLECLOUD_PROJECT_PATH}/default/files/hwlocs-install.sh" || fail + +ROLE=$(jetpack config pbspro.role "") || fail + +echo "jetpack config pbspro.role $ROLE" + +case "$ROLE" in + server) bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/server-install.sh" || fail ;; + login) bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/login-install.sh" || fail ;; + execute) bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/execute-install.sh" || fail ;; + *) fail "Unknown pbspro.role '$ROLE'" ;; +esac diff --git a/specs/default/cluster-init/roles/execute-install.sh b/specs/default/cluster-init/roles/execute-install.sh new file mode 100755 index 0000000..afe34b1 --- /dev/null +++ b/specs/default/cluster-init/roles/execute-install.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 + +EXECUTE_HOSTNAME=$(jetpack config hostname) || fail +PACKAGE_NAME=$(get_package_name "execution") || fail +SERVER_HOSTNAME=$(get_server_hostname) || fail + +# Forces execute node's hostname to be updated (scalelib is blocked until the hostname is correct) +# TODO: this installation status should be done by jetpack before cluster-inits are run +"${CYCLECLOUD_HOME}/system/embedded/bin/python" -c "import jetpack.converge as jc; jc._send_installation_status('warning')" + +jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail +yum install -y -q "/tmp/$PACKAGE_NAME" || fail + +if [[ -n "$SERVER_HOSTNAME" ]]; then + echo "$SERVER_HOSTNAME" > /var/spool/pbs/server_name + chmod 0644 /var/spool/pbs/server_name || fail + + cp "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/mom_config.template" /var/spool/pbs/mom_priv/config || fail + chmod 0644 /var/spool/pbs/mom_priv/config || fail + + sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ + "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail + chmod 0644 /etc/pbs.conf || fail +fi + +await_node_definition() { + if ! /opt/pbs/bin/pbsnodes "$EXECUTE_HOSTNAME"; then + echo "${EXECUTE_HOSTNAME} is not in the cluster yet. Retrying next converge" 1>&2 + return 1 + fi +} + +readonly MAX_RETRIES=10 +readonly RETRY_DELAY=15 +ATTEMPT=1 +if ! await_node_definition; then + while [[ $ATTEMPT -lt $MAX_RETRIES ]]; do + sleep $RETRY_DELAY + ((ATTEMPT+=1)) + + if await_node_definition; then + break; + fi + done + + if [[ $ATTEMPT == $MAX_RETRIES ]]; then + fail "Command failed after $MAX_RETRIES attempts. Exiting." + fi +fi + +# This block will execute only if the "execute" node is defined in the PBS server +NODE_CREATED_GUARD="pbs.nodecreated" +if [[ -f "$NODE_CREATED_GUARD" ]]; then + echo "Node has already been created, skipping joining checks" +else + NODE_ATTRS=$(/opt/pbs/bin/pbsnodes "$EXECUTE_HOSTNAME") || fail + NODE_ID=$(jetpack config cyclecloud.node.id) || fail + if ! echo "$NODE_ATTRS" | bool grep -qi "$NODE_ID"; then + fail "Stale entry found for $EXECUTE_HOSTNAME. Waiting for autoscaler to update this before joining." + fi + + /opt/pbs/bin/pbsnodes -o "$EXECUTE_HOSTNAME" -C 'cyclecloud offline' || fail + + touch "$NODE_CREATED_GUARD" || fail +fi \ No newline at end of file diff --git a/specs/default/cluster-init/roles/login-install.sh b/specs/default/cluster-init/roles/login-install.sh new file mode 100755 index 0000000..1379a65 --- /dev/null +++ b/specs/default/cluster-init/roles/login-install.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 + +PACKAGE_NAME=$(get_package_name "client") || fail +SERVER_HOSTNAME=$(get_server_hostname) || fail + +jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail +yum install -y -q "/tmp/$PACKAGE_NAME" || fail + +if [[ -n "$SERVER_HOSTNAME" ]]; then + sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ + "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail + chmod 0644 /etc/pbs.conf || fail +fi + +/opt/pbs/bin/qmgr -c "set server flatuid=true" || fail \ No newline at end of file diff --git a/specs/default/cluster-init/roles/server-install.sh b/specs/default/cluster-init/roles/server-install.sh new file mode 100644 index 0000000..5b67ac8 --- /dev/null +++ b/specs/default/cluster-init/roles/server-install.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 +source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail +echo "hi im the server node" +PACKAGE_NAME=$(get_package_name "server") || fail +CLUSTER_NAME=$(jq -r .cluster "$CONFIG_PATH") || fail +CONNECTION_URL=$(jq -r .url "$CONFIG_PATH") || fail +IGNORE_WORKQ=$(jetpack config pbspro.queues.workq.ignore "False") || fail +IGNORE_HTCQ=$(jetpack config pbspro.queues.htcq.ignore "False") || fail +CRON_METHOD=$(jetpack config pbspro.cron_method "pbs_cron") || fail +PBSPRO_AUTOSCALE_PROJECT_HOME="/opt/cycle/pbspro" +PBSPRO_AUTOSCALE_INSTALLER="cyclecloud-pbspro-pkg-${PBSPRO_AUTOSCALE_VERSION}.tar.gz" + +mkdir -p "/sched/${CLUSTER_NAME}" || fail + +cat << EOF > "/sched/${CLUSTER_NAME}/azpbs.env" +#!/bin/bash +PBS_SCHEDULER_HOSTNAME=$(hostname) +PBS_SCHEDULER_IP=$(hostname -i) + +EOF +chmod a+r "/sched/${CLUSTER_NAME}/azpbs.env" || fail + +jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail +yum install -y -q "/tmp/$PACKAGE_NAME" || fail + +mkdir -p -m 0755 "$PBSPRO_AUTOSCALE_PROJECT_HOME" || fail + +mkdir -p -m 750 /var/spool/pbs/sched_priv || fail + +cp "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/sched.config" /var/spool/pbs/sched_priv/sched_config || fail +chmod 0644 /var/spool/pbs/sched_priv/sched_config || fail + +systemctl enable --now pbs || fail + +source /etc/profile.d/pbs.sh || fail +PATH=$PATH:/root/bin + +cd "$BOOTSTRAP_HOME" || fail # TODO: find a new location instead of BOOTSTRAP_HOME + +rm -f "$PBSPRO_AUTOSCALE_INSTALLER" || fail + +jetpack download "$PBSPRO_AUTOSCALE_INSTALLER" --project pbspro ./ || fail + +if [ -e cyclecloud-pbspro ]; then + rm -rf cyclecloud-pbspro/ || fail +fi + +tar xzf "$PBSPRO_AUTOSCALE_INSTALLER" || fail + +cd cyclecloud-pbspro/ || fail + +INSTALLDIR=$(realpath "$PBSPRO_AUTOSCALE_PROJECT_HOME") || fail +mkdir -p "${INSTALLDIR}/venv" || fail + +IGNORE_QUEUES_ARG="" +if [[ "$IGNORE_WORKQ" == "True" && "$IGNORE_HTCQ" == "True" ]]; then + IGNORE_QUEUES_ARG="--ignore-queues workq,htcq" +elif [[ "$IGNORE_WORKQ" == "True" ]]; then + IGNORE_QUEUES_ARG="--ignore-queues workq" +elif [[ "$IGNORE_HTCQ" == "True" ]]; then + IGNORE_QUEUES_ARG="--ignore-queues htcq" +fi + +./initialize_pbs.sh || fail + +./initialize_default_queues.sh || fail + +./install.sh --install-python3 --venv "${INSTALLDIR}/venv" --cron-method "$CRON_METHOD" || fail + +./generate_autoscale_json.sh --install-dir "$INSTALLDIR" \ + --username "$(jetpack config cyclecloud.config.username)" \ + --password "$(jetpack config cyclecloud.config.password)" \ + --url "$CONNECTION_URL" \ + --cluster-name "$CLUSTER_NAME" \ + $IGNORE_QUEUES_ARG \ + || fail + +ls "${PBSPRO_AUTOSCALE_PROJECT_HOME}/autoscale.json" || fail +azpbs connect || fail + +systemctl restart pbs || fail \ No newline at end of file From 1bab8459f032c292da1937a82e2e3797ca5329a2 Mon Sep 17 00:00:00 2001 From: Ryan Hamel Date: Fri, 24 Jul 2026 10:42:57 -0400 Subject: [PATCH 02/18] use almalinux:8 for the docker build --- docker-rpmbuild.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-rpmbuild.sh b/docker-rpmbuild.sh index 30d74ec..41c3fdc 100755 --- a/docker-rpmbuild.sh +++ b/docker-rpmbuild.sh @@ -1,7 +1,7 @@ #!/bin/bash if command -v docker; then - docker run -v $(pwd)/specs/default/cluster-init/files:/source -v $(pwd)/blobs:/root/rpmbuild/RPMS/x86_64 -ti centos:7 /bin/bash -e /source/00-build-pbspro.sh + docker run -v $(pwd)/specs/default/cluster-init/files:/source -v $(pwd)/blobs:/root/rpmbuild/RPMS/x86_64 -ti almalinux:8 /bin/bash -e /source/00-build-pbspro.sh else echo "`docker` binary not found. Install docker to build RPMs with this script" fi From 1f0beb2304bc69d5104ab40b07522e9a1d99a157 Mon Sep 17 00:00:00 2001 From: Ryan Hamel Date: Fri, 24 Jul 2026 11:36:50 -0400 Subject: [PATCH 03/18] Local docker build and use almalinux8 for release build --- .github/workflows/release.yml | 11 +++-- build.sh | 7 +-- docker-package.sh | 85 +++++++++++++++++++++++++++++++++++ package.py | 2 +- package.sh | 27 +++++++++++ util/Dockerfile | 4 ++ util/build.sh | 29 ++++++++++++ 7 files changed, 157 insertions(+), 8 deletions(-) create mode 100755 docker-package.sh create mode 100755 package.sh create mode 100644 util/Dockerfile create mode 100755 util/build.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac8d5a6..e37a2f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,16 +10,19 @@ jobs: build: name: Upload Release Asset runs-on: ubuntu-latest + container: + image: almalinux:8 permissions: contents: write steps: + - name: Install build prerequisites + run: | + dnf update -y + dnf install -y python3.11 python3.11-pip git tar gzip which findutils - name: Checkout code uses: actions/checkout@v4 - name: Build pkg - run: | - sudo apt update || apt update - sudo apt-get install -y python3 python3-pip || apt-get install -y python3 python3-pip - ./build.sh + run: ./build.sh - name: Get the version id: get_version run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT diff --git a/build.sh b/build.sh index 6719837..aaeeae4 100755 --- a/build.sh +++ b/build.sh @@ -1,11 +1,12 @@ #!/bin/bash # create a new venv if it does not exist, or is older than 7 days -if [ ! $(find . -path ./venv/created -mtime -7) ]; then +if [ -z "$(find . -path ./venv/created -mtime -7 -print -quit)" ] || + ! venv/bin/python -c 'import sys; raise SystemExit(sys.version_info < (3, 11))' 2>/dev/null; then rm -rf venv - python3 -m venv venv + python3.11 -m venv venv source venv/bin/activate - pip install setuptools + python -m pip install setuptools touch venv/created else source venv/bin/activate diff --git a/docker-package.sh b/docker-package.sh new file mode 100755 index 0000000..8f5f0b2 --- /dev/null +++ b/docker-package.sh @@ -0,0 +1,85 @@ +#!/bin/bash +echo see .build.log for more information +log_file=".build.log" +check_dirty_changes() { + if [ -n "$(git status --porcelain)" ]; then + echo "Error: There are uncommitted changes in the current branch. Please commit or stash them before running this script." + exit 1 + fi +} + +# Call the function to check for dirty changes +# check_dirty_changes + +print_branch_and_last_commit() { + branch=$(git rev-parse --abbrev-ref HEAD) + last_commit=$(git log -1 --pretty=format:"%h - %s (%ci)") + + printf "%-20s: %s\n" "Current branch" "$branch" + printf "%-20s: %s\n" "Last commit" "$last_commit" +} + +delete_existing_blobs() { + printf "%-20s: %s\n" "Deleting existing Blob Files" "" + while IFS= read -r file; do + if [ -f "blobs/$file" ]; then + printf "%-20s: %s\n" "" "$file" + rm -f "blobs/$file" || exit 1 + fi + done < <(awk -F' *= *' '/^\[blobs\]/ {found=1} found && /^Files/ {gsub(/, */, "\n", $2); print $2; exit}' project.ini) +} + +check_blobs_files_exist() { + local version="$1" + local missing_files=0 + + printf "%-20s: %s\n" "Blob Files" "" + while IFS= read -r file; do + printf "%-20s: %s\n" "" "$file" + if [ ! -f "blobs/$file" ]; then + echo "Error: File blobs/$file does not exist." + missing_files=1 + fi + done < <(awk -F' *= *' '/^\[blobs\]/ {found=1} found && /^Files/ {gsub(/, */, "\n", $2); print $2; exit}' project.ini) + + if [ $missing_files -eq 1 ]; then + echo "One or more required files are missing in the blobs directory." + exit 1 + + fi +} + +get_version_from_project_ini() { + version=$(awk -F' *= *' '/^\[project\]/ {found=1} found && /^version/ {print $2; exit}' project.ini) + printf "%-20s: %s\n" "Project Version" "$version" + check_blobs_files_exist "$version" +} + +local_azpbs=/source/ +if [ "$1" != "" ]; then + scalelib=$(realpath $1) + local_scalelib=/source/cyclecloud-scalelib + extra_args="-v ${scalelib}:${local_scalelib}" +fi + +if command -v docker; then + runtime=docker + runtime_args= +elif command -v podman; then + runtime=podman + runtime_args="--privileged" +else + echo "`docker` or `podman` binary not found. Install docker or podman to build RPMs with this script" + exit 1 +fi + +{ + delete_existing_blobs + # allows caching + $runtime build -t azpbs_build:latest -f util/Dockerfile . + $runtime run -v $(pwd):${local_azpbs} $runtime_args $extra_args -ti azpbs_build:latest /bin/bash ${local_azpbs}/util/build.sh $local_scalelib +} &> $log_file + +# Call the function to print the branch and the last commit +print_branch_and_last_commit +get_version_from_project_ini diff --git a/package.py b/package.py index d917b8f..041255a 100644 --- a/package.py +++ b/package.py @@ -119,7 +119,7 @@ def _add(name: str, path: Optional[str] = None, mode: Optional[int] = None) -> N _add("packages/" + dep, dep_path) packages.append(dep_path) - check_call(["pip", "download"] + packages, cwd=build_dir) + check_call([sys.executable, "-m", "pip", "download"] + packages, cwd=build_dir) print("Using build dir", build_dir) by_package: Dict[str, List[str]] = {} diff --git a/package.sh b/package.sh new file mode 100755 index 0000000..20eedf1 --- /dev/null +++ b/package.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e +cd $(dirname $0)/ + + +if [ ! -e libs ]; then + mkdir libs +fi + +LOCAL_SCALELIB=$1 + +rm -f dist/* + +if [ "$LOCAL_SCALELIB" == "" ]; then + # we are using released versions of scalelib + python3.11 package.py +else + pushd $LOCAL_SCALELIB + rm -f dist/*.gz + # python3 setup.py swagger + python3.11 setup.py sdist + popd + # swagger=`ls $LOCAL_SCALELIB/dist/swagger*.gz` + scalelib=`ls $LOCAL_SCALELIB/dist/cyclecloud_scalelib*.gz` + # python3 package.py --scalelib $scalelib --swagger $swagger + python3.11 package.py --scalelib $scalelib +fi diff --git a/util/Dockerfile b/util/Dockerfile new file mode 100644 index 0000000..d4cf5e0 --- /dev/null +++ b/util/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.11 +RUN pip3.11 install --upgrade setuptools +RUN pip3.11 install --upgrade wheel +RUN pip3.11 install requests \ No newline at end of file diff --git a/util/build.sh b/util/build.sh new file mode 100755 index 0000000..89afb68 --- /dev/null +++ b/util/build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -e +set -x + +if [ "$1" == "-h" ] || [ "$1" == "--help" ] || [ "$1" == "-help" ]; then + echo "Usage: $0 [path/to/scalelib repo]" + echo "If no path to scalelib is passed in, one will be downloaded from GitHub based on" + echo "the version specified in package.py:SCALELIB_VERSION" + exit 1 +fi + +LOCAL_SCALELIB=$1 + +if [ "$LOCAL_SCALELIB" != "" ]; then + LOCAL_SCALELIB=$(realpath $LOCAL_SCALELIB) +fi + +cwd=$(dirname "$(readlink -f "$0")") +SOURCE=$(dirname $cwd) + +if [ ! -e $SOURCE/blobs ]; then + mkdir $SOURCE/blobs +fi + + +cd $SOURCE +rm -f dist/* +./package.sh +mv dist/* blobs/ From 5f64e9e76579937923a0a93436a8b7e80d96bcd6 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Wed, 19 Aug 2026 20:22:26 +0000 Subject: [PATCH 04/18] Use python3.11, use newest cyclecloud api wheel, and replace username + password parameters for autoscale script --- generate_autoscale_json.sh | 4 +++- install.sh | 24 +++++++++++++------ package.py | 4 ++-- project.ini | 2 +- .../cluster-init/roles/server-install.sh | 2 -- util.py | 2 +- 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/generate_autoscale_json.sh b/generate_autoscale_json.sh index 0a152ab..d06f0f3 100755 --- a/generate_autoscale_json.sh +++ b/generate_autoscale_json.sh @@ -28,7 +28,7 @@ CLUSTER_NAME= IGNORE_QUEUES_ARG= function usage() { - echo Usage: $0 --username username --password password --url https://fqdn:port --cluster-name cluster_name [--install-dir /opt/cycle/pbspro] + echo Usage: $0 --url https://fqdn:port --cluster-name cluster_name [--username username] [--password password] [--install-dir /opt/cycle/pbspro] exit 2 } @@ -73,7 +73,9 @@ done if [ "$1" == "-h" ]; then usage; fi if [ "$1" == "-help" ]; then usage; fi +if [ "$USERNAME" == "" ]; then USERNAME=$(jetpack config cyclecloud.config.username); fi if [ "$USERNAME" == "" ]; then usage; fi +if [ "$PASSWORD" == "" ]; then PASSWORD=$(jetpack config cyclecloud.config.password); fi if [ "$PASSWORD" == "" ]; then usage; fi if [ "$URL" == "" ]; then usage; fi if [ "$CLUSTER_NAME" == "" ]; then usage; fi diff --git a/install.sh b/install.sh index 58e53c3..22b2ae2 100755 --- a/install.sh +++ b/install.sh @@ -56,27 +56,37 @@ echo VENV=$VENV # remove jetpack's python3 from the path export PATH=$(echo "$PATH" | sed -e 's/\/opt\/cycle\/jetpack\/system\/embedded\/bin://g' | sed -e 's/:\/opt\/cycle\/jetpack\/system\/embedded\/bin//g') set +e -which python3 > /dev/null; +which python3.11 > /dev/null; if [ $? != 0 ]; then if [ $INSTALL_PYTHON3 == 1 ]; then - yum install -y -q python3 || exit 1 + yum install -y -q python3.11 || exit 1 else - echo Please install python3 >&2; + echo Please install python3.11 >&2; + exit 1 + fi +fi + +python3.11 -m pip --version > /dev/null 2>&1 +if [ $? != 0 ]; then + if [ $INSTALL_PYTHON3 == 1 ]; then + yum install -y -q python3.11-pip || exit 1 + else + echo Please install pip for python3.11 >&2; exit 1 fi fi set -e if [ $INSTALL_VIRTUALENV == 1 ]; then - python3 -m pip install -q virtualenv + python3.11 -m pip install -q virtualenv fi set +e -python3 -m virtualenv --version 2>&1 > /dev/null +python3.11 -m virtualenv --version 2>&1 > /dev/null if [ $? != 0 ]; then if [ $INSTALL_VIRTUALENV ]; then - python3 -m pip install -q virtualenv || exit 1 + python3.11 -m pip install -q virtualenv || exit 1 else echo Please install virtualenv for python3 >&2 exit 1 @@ -84,7 +94,7 @@ if [ $? != 0 ]; then fi set -e -python3 -m virtualenv $VENV +python3.11 -m virtualenv $VENV source "${VENV}/bin/activate" # not sure why but pip gets confused installing frozendict locally # if you don't install it first. It has no dependencies so this is safe. diff --git a/package.py b/package.py index 041255a..c7e8456 100644 --- a/package.py +++ b/package.py @@ -12,7 +12,7 @@ from util import download_release_files SCALELIB_VERSION = "1.0.11" -CYCLECLOUD_API_VERSION = "8.3.1" +CYCLECLOUD_API_VERSION = "8.10.0" def build_sdist() -> str: @@ -39,7 +39,7 @@ def get_cycle_packages(args: Namespace) -> List[str]: scalelib_url = f"https://github.com/Azure/cyclecloud-scalelib/archive/refs/tags/{SCALELIB_VERSION}.tar.gz" - cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/{cyclecloud_api_file}" + cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/{cyclecloud_api_file}" #TODO: ensure this is the correct url to_download = { scalelib_file: (args.scalelib, scalelib_url), cyclecloud_api_file: (args.cyclecloud_api, cyclecloud_api_url), diff --git a/project.ini b/project.ini index 35e0a24..3c3539a 100644 --- a/project.ini +++ b/project.ini @@ -6,7 +6,7 @@ version = 2.0.26 autoupgrade = true [blobs] -Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, cyclecloud_api-8.3.1-py2.py3-none-any.whl, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm +Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, cyclecloud_api-8.10.0-py2.py3-none-any.whl, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm [spec server] diff --git a/specs/default/cluster-init/roles/server-install.sh b/specs/default/cluster-init/roles/server-install.sh index 5b67ac8..3544e48 100644 --- a/specs/default/cluster-init/roles/server-install.sh +++ b/specs/default/cluster-init/roles/server-install.sh @@ -70,8 +70,6 @@ fi ./install.sh --install-python3 --venv "${INSTALLDIR}/venv" --cron-method "$CRON_METHOD" || fail ./generate_autoscale_json.sh --install-dir "$INSTALLDIR" \ - --username "$(jetpack config cyclecloud.config.username)" \ - --password "$(jetpack config cyclecloud.config.password)" \ --url "$CONNECTION_URL" \ --cluster-name "$CLUSTER_NAME" \ $IGNORE_QUEUES_ARG \ diff --git a/util.py b/util.py index d5ef50c..b502a2b 100644 --- a/util.py +++ b/util.py @@ -10,7 +10,7 @@ def download_release_files(): blobs = get_blobs() for _, fname in enumerate(blobs): - if fname == "cyclecloud_api": + if "cyclecloud_api" in fname: continue url = os.path.join(RELEASE_URL, fname) From dfcaac3c929fa031662c695012b27b4a4680ada8 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Fri, 21 Aug 2026 19:19:18 +0000 Subject: [PATCH 05/18] Configure tests to work for default spec --- .gitignore | 1 + .../cluster-init/tests/helper.py | 0 .../cluster-init/tests/test_execute.py | 5 ++++- .../cluster-init/tests/test_submit.py | 3 ++- .../cluster-init/tests/tryme.py | 0 .../login/cluster-init/tests/test_execute.py | 19 ------------------- templates/openpbs.txt | 5 +++++ 7 files changed, 12 insertions(+), 21 deletions(-) rename specs/{server => default}/cluster-init/tests/helper.py (100%) rename specs/{execute => default}/cluster-init/tests/test_execute.py (77%) rename specs/{server => default}/cluster-init/tests/test_submit.py (98%) rename specs/{server => default}/cluster-init/tests/tryme.py (100%) delete mode 100644 specs/login/cluster-init/tests/test_execute.py diff --git a/.gitignore b/.gitignore index 0174ffc..5741ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ dist/** .env libs/** venv/** +.build.log diff --git a/specs/server/cluster-init/tests/helper.py b/specs/default/cluster-init/tests/helper.py similarity index 100% rename from specs/server/cluster-init/tests/helper.py rename to specs/default/cluster-init/tests/helper.py diff --git a/specs/execute/cluster-init/tests/test_execute.py b/specs/default/cluster-init/tests/test_execute.py similarity index 77% rename from specs/execute/cluster-init/tests/test_execute.py rename to specs/default/cluster-init/tests/test_execute.py index 335684d..dabdd23 100644 --- a/specs/execute/cluster-init/tests/test_execute.py +++ b/specs/default/cluster-init/tests/test_execute.py @@ -3,8 +3,11 @@ # import unittest import subprocess +import jetpack - +PBSPRO_ROLE = jetpack.config.get("pbspro.role", None) + +@unittest.skipUnless(PBSPRO_ROLE == "execute" or PBSPRO_ROLE == "login", "execute or login-only test") class TestExecute(unittest.TestCase): def test_simple(self): diff --git a/specs/server/cluster-init/tests/test_submit.py b/specs/default/cluster-init/tests/test_submit.py similarity index 98% rename from specs/server/cluster-init/tests/test_submit.py rename to specs/default/cluster-init/tests/test_submit.py index 79d2dc4..4d408b3 100644 --- a/specs/server/cluster-init/tests/test_submit.py +++ b/specs/default/cluster-init/tests/test_submit.py @@ -19,6 +19,7 @@ logger = logging.getLogger() CLUSTER_USER = props.get("cyclecloud.owner") +PBSPRO_ROLE = jetpack.config.get("pbspro.role", None) def readfile_if_exist(filename): @@ -47,7 +48,7 @@ def write_job_script(): return job_script - +@unittest.skipUnless(PBSPRO_ROLE == "server", "server-only test") class TestSubmit(unittest.TestCase): def setUp(self): diff --git a/specs/server/cluster-init/tests/tryme.py b/specs/default/cluster-init/tests/tryme.py similarity index 100% rename from specs/server/cluster-init/tests/tryme.py rename to specs/default/cluster-init/tests/tryme.py diff --git a/specs/login/cluster-init/tests/test_execute.py b/specs/login/cluster-init/tests/test_execute.py deleted file mode 100644 index 335684d..0000000 --- a/specs/login/cluster-init/tests/test_execute.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -import unittest -import subprocess - - -class TestExecute(unittest.TestCase): - - def test_simple(self): - p = subprocess.Popen(['/opt/pbs/bin/qstat'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - if hasattr(stdout, "decode"): - stdout = stdout.decode() - stderr = stderr.decode() - - self.assertEqual(0, p.returncode, msg="Call to qstat failed with Stderr: %s\nStdout%s" - % (stderr, stdout)) - diff --git a/templates/openpbs.txt b/templates/openpbs.txt index 5303d1d..97373c1 100644 --- a/templates/openpbs.txt +++ b/templates/openpbs.txt @@ -52,11 +52,13 @@ Autoscale = $Autoscale export_path = $NFSSharedExportPath address = $NFSAddress options = $NFSSharedMountOptions + export_name = server [[[configuration cyclecloud.mounts.nfs_sched]]] type = nfs mountpoint = /sched disabled = $NFSSchedDisable + export_name = server [[[configuration cyclecloud.mounts.additional_nfs]]] disabled = ${AdditionalNAS isnt true} @@ -80,6 +82,7 @@ Autoscale = $Autoscale pbspro.cron_method = $AzpbsCronMethod pbspro.queues.workq.ignore = ${Autoscale != true} pbspro.queues.htcq.ignore = ${Autoscale != true} + pbspro.role = server [[[cluster-init cyclecloud/pbspro:server]]] @@ -129,6 +132,7 @@ Autoscale = $Autoscale [[[configuration]]] autoscale.enabled = false + pbspro.role = login [[nodearray execute]] MachineType = $ExecuteMachineType @@ -145,6 +149,7 @@ Autoscale = $Autoscale # {"pbspro": {"ignore_queues": ["workq", "htcq"]}} # which is what pbspro.queues.workq.ignore does as well. # autoscale.enabled = $Autoscale + pbspro.role = execute [[[cluster-init cyclecloud/pbspro:execute]]] From 08acd1ecbf29c083144b66604f212c8fa27bcf5b Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Mon, 24 Aug 2026 22:13:45 +0000 Subject: [PATCH 06/18] Modify template to match new file system options from Slurm --- templates/openpbs.txt | 226 ++++++++++++++++++++++++++++++------------ 1 file changed, 165 insertions(+), 61 deletions(-) diff --git a/templates/openpbs.txt b/templates/openpbs.txt index 97373c1..214fb21 100644 --- a/templates/openpbs.txt +++ b/templates/openpbs.txt @@ -17,6 +17,10 @@ Autoscale = $Autoscale Region = $Region KeyPairLocation = ~/.ssh/cyclecloud.pem Azure.Identities = $ManagedIdentity + + # Lustre mounts require termination notifications to unmount + EnableTerminateNotification = ${NFSType == "lustre" || NFSSchedType == "lustre" || AdditionalNFSType == "lustre" || EnableTerminateNotification} + TerminateNotificationTimeout = 10m [[[configuration]]] pbspro.version = $PBSVersion @@ -47,25 +51,27 @@ Autoscale = $Autoscale SSD = True [[[configuration cyclecloud.mounts.nfs_shared]]] - type = nfs + type = $NFSType mountpoint = /shared - export_path = $NFSSharedExportPath + export_path = ${ifThenElse(NFSType == "lustre", strcat("tcp:/lustrefs", NFSSharedExportPath), NFSSharedExportPath)} address = $NFSAddress options = $NFSSharedMountOptions export_name = server [[[configuration cyclecloud.mounts.nfs_sched]]] - type = nfs + type = $NFSSchedType mountpoint = /sched - disabled = $NFSSchedDisable + export_path = ${ifThenElse(NFSSchedType == "lustre", strcat("tcp:/lustrefs", NFSSchedExportPath), NFSSchedExportPath)} + address = ${ifThenElse(UseBuiltinSched, undefined, NFSSchedAddress)} + options = $NFSSchedMountOptions export_name = server [[[configuration cyclecloud.mounts.additional_nfs]]] - disabled = ${AdditionalNAS isnt true} - type = nfs - address = $AdditonalNFSAddress + disabled = ${AdditionalNFS isnt true} + type = $AdditionalNFSType + address = $AdditionalNFSAddress mountpoint = $AdditionalNFSMountPoint - export_path = $AdditionalNFSExportPath + export_path = ${ifThenElse(AdditionalNFSType == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath), AdditionalNFSExportPath)} options = $AdditionalNFSMountOptions @@ -77,8 +83,8 @@ Autoscale = $Autoscale [[[configuration]]] cyclecloud.discoverable=true - cyclecloud.mounts.nfs_sched.disabled = true - cyclecloud.mounts.nfs_shared.disabled = ${NFSType != "External"} + cyclecloud.mounts.nfs_sched.disabled = $UseBuiltinSched + cyclecloud.mounts.nfs_shared.disabled = $UseBuiltinShared pbspro.cron_method = $AzpbsCronMethod pbspro.queues.workq.ignore = ${Autoscale != true} pbspro.queues.htcq.ignore = ${Autoscale != true} @@ -90,34 +96,38 @@ Autoscale = $Autoscale AssociatePublicIpAddress = $UsePublicNetwork [[[volume sched]]] - Size = 1024 + Size = $SchedFilesystemSize SSD = True Mount = builtinsched - Persistent = False + Persistent = True #TODO: does this need to be true + Disabled = ${!UseBuiltinSched} [[[volume shared]]] - Size = ${ifThenElse(NFSType == "Builtin", FilesystemSize, 2)} + Size = $FilesystemSize SSD = True Mount = builtinshared - Persistent = ${NFSType == "Builtin"} + Persistent = True True #TODO: does this need to be true + Disabled = ${!UseBuiltinShared} #TODO: test new variable [[[configuration cyclecloud.mounts.builtinsched]]] + disabled = ${!UseBuiltinSched} mountpoint = /sched fs_type = xfs [[[configuration cyclecloud.mounts.builtinshared]]] - disabled = ${NFSType != "Builtin"} + disabled = ${!UseBuiltinShared} mountpoint = /shared fs_type = xfs [[[configuration cyclecloud.exports.builtinsched]]] + disabled = ${!UseBuiltinSched} export_path = /sched options = no_root_squash samba.enabled = false type = nfs [[[configuration cyclecloud.exports.builtinshared]]] - disabled = ${NFSType != "Builtin"} + disabled = ${!UseBuiltinShared} export_path = /shared samba.enabled = false type = nfs @@ -237,61 +247,139 @@ Order = 10 [parameters Network Attached Storage] Order = 15 + [[parameters Shared Storage]] + Order = 10 + + [[[parameter About Shared Storage]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

The directories /sched and /shared are network attached mounts and exist on all nodes of the cluster.
+
+ Options for providing these mounts:
+ [Builtin]: The server node is an NFS server that provides the mountpoint to the other nodes of the cluster.
+ [External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server provides the mountpoint.
+ [Azure Managed Lustre]: An Azure Managed Lustre deployment provides the mountpoint.
+

+

Note: the cluster must be terminated for changes to filesystem mounts to take effect.

''' + Conditions.Hidden := false [[parameters Scheduler Mount]] - Order = 5 + Order = 20 + Label = File-system Mount for /sched + [[[parameter About sched]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template = '''

The directory /sched is a network attached mount and exists in all nodes of the cluster. - It's managed by the scheduler node. - To disable the mount of the /sched directory, and to supply your own for a hybrid scenario, select the checkbox below.''' + Config.Template = '''

OpenPBS configuration shared with cluster nodes is stored in the /sched directory. It is managed by the server node.

''' Order = 6 - [[[parameter NFSSchedDisable]]] + [[[parameter About sched part 2]]] HideLabel = true - DefaultValue = false - Widget.Plugin = pico.form.BooleanCheckBox - Widget.Label = External Scheduler + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Uncheck the box below to disable the built-in NFS export of the /sched directory and use an external file-system.

''' + Order = 7 - [[parameters Default NFS Share]] - Order = 10 - [[[parameter About shared]]] + [[[parameter UseBuiltinSched]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /sched + DefaultValue = true + ParameterType = Boolean + + [[[parameter NFSSchedDiskWarning]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

The directory /shared is a network attached mount and exists in all nodes of the cluster. Users' home directories reside within this mountpoint with the base homedir /shared/home.

There are two options for providing this mount:
[Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster.
[External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server, provides the mountpoint.

Note: the cluster must be terminated for this to take effect.

" - Order = 20 + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the scheduler disk.

" + Conditions.Hidden := UseBuiltinSched - [[[parameter NFSType]]] - Label = NFS Type + [[[parameter NFSSchedType]]] + Label = FS Type ParameterType = StringList - Config.Label = Type of NFS to use for this cluster + Config.Label = Type of shared filesystem to use for this cluster Config.Plugin = pico.form.Dropdown - Config.Entries := {[Label="External NFS"; Value="External"], [Label="Builtin"; Value="Builtin"]} - DefaultValue = Builtin + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedAddress]]] + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedExportPath]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /sched + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedMountOptions]]] + Label = Mount Options + Description = File system client mount options + Conditions.Hidden := UseBuiltinSched + + [[[parameter SchedFilesystemSize]]] + Label = Size (GB) + Description = The filesystem size (cannot be changed after initial start) + DefaultValue = 1024 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := !UseBuiltinSched + + [[parameters Default NFS Share]] + Order = 30 + Label = File-system Mount for /shared + + [[[parameter About shared]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Users' home directories reside within the /shared mountpoint with the base homedir /shared/home.

''' + Order = 6 + + [[[parameter About shared part 2]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Uncheck the box below to disable the built-in NFS export of the /shared directory and use an external file-system.

''' + Order = 7 + + [[[parameter UseBuiltinShared]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /shared + DefaultValue = true + ParameterType = Boolean [[[parameter NFSDiskWarning]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Warning: switching an active cluster over to NFS will delete the shared disk.

" - Conditions.Hidden := NFSType != "External" + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinShared + + [[[parameter NFSType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Type of filesystem to use for /shared + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinShared [[[parameter NFSAddress]]] - Label = NFS IP Address - Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String - Conditions.Hidden := NFSType != "External" + Conditions.Hidden := UseBuiltinShared [[[parameter NFSSharedExportPath]]] - Label = Shared Export Path + Label = Export Path Description = The path exported by the file system DefaultValue = /shared - Conditions.Hidden := NFSType != "External" + Conditions.Hidden := UseBuiltinShared [[[parameter NFSSharedMountOptions]]] - Label = NFS Mount Options - Description = NFS Client Mount Options - Conditions.Hidden := NFSType != "External" + Label = Mount Options + Description = File system client mount options + Conditions.Hidden := UseBuiltinShared [[[parameter FilesystemSize]]] Label = Size (GB) @@ -302,44 +390,56 @@ Order = 15 Config.MinValue = 10 Config.MaxValue = 10240 Config.IntegerOnly = true - Conditions.Excluded := NFSType != "Builtin" + Conditions.Excluded := !UseBuiltinShared [[parameters Additional NFS Mount]] - Order = 20 - [[[parameter Additional NFS Mount Readme]]] + Order = 40 + Label = Additional Filesystem Mount + + [[[parameter Additional Shared FS Mount Readme]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Mount another NFS endpoint on the cluster nodes.

" + Config.Template := "

Mount another shared file-system endpoint on the cluster nodes.

" Order = 20 - [[[parameter AdditionalNAS]]] + [[[parameter AdditionalNFS]]] HideLabel = true DefaultValue = false Widget.Plugin = pico.form.BooleanCheckBox - Widget.Label = Add NFS mount + Widget.Label = Add Shared Filesystem mount + + [[[parameter AdditionalNFSType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Shared filesystem type of the additional mount + Config.Plugin = pico.form.Dropdown - [[[parameter AdditonalNFSAddress]]] - Label = NFS IP Address - Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Excluded := AdditionalNFS isnt true + + [[[parameter AdditionalNFSAddress]]] + Label = IP Address + Description = The IP address or hostname of the additional mount. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSMountPoint]]] - Label = NFS Mount Point + Label = Mount Point Description = The path at which to mount the Filesystem DefaultValue = /data - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSExportPath]]] - Label = NFS Export Path + Label = Export Path Description = The path exported by the file system DefaultValue = /data - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSMountOptions]]] - Label = NFS Mount Options - Description = NFS Client Mount Options - Conditions.Excluded := AdditionalNAS isnt true + Label = Mount Options + Description = Filesystem Client Mount Options + Conditions.Excluded := AdditionalNFS isnt true @@ -369,6 +469,10 @@ Order = 20 Config.Increment = 64 DefaultValue = 0 + [[[parameter EnableTerminateNotification]]] + Label = Enable Termination notifications + DefaultValue = false + [[parameters Software]] From 90e7ac858abbf7fdd6d9c1573f5a155291606b3d Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Fri, 4 Sep 2026 22:12:34 +0000 Subject: [PATCH 07/18] Run Docker container as host developer to prevent root-owned build artifacts and through build.sh --- build.sh | 8 ++++---- docker-package.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index aaeeae4..536c4cc 100755 --- a/build.sh +++ b/build.sh @@ -1,16 +1,16 @@ #!/bin/bash +set -e # create a new venv if it does not exist, or is older than 7 days if [ -z "$(find . -path ./venv/created -mtime -7 -print -quit)" ] || ! venv/bin/python -c 'import sys; raise SystemExit(sys.version_info < (3, 11))' 2>/dev/null; then rm -rf venv - python3.11 -m venv venv + python3 -m venv venv source venv/bin/activate - python -m pip install setuptools + python3 -m pip install setuptools touch venv/created else source venv/bin/activate fi -python package.py - +./docker-package.sh \ No newline at end of file diff --git a/docker-package.sh b/docker-package.sh index 8f5f0b2..15bca3a 100755 --- a/docker-package.sh +++ b/docker-package.sh @@ -77,7 +77,7 @@ fi delete_existing_blobs # allows caching $runtime build -t azpbs_build:latest -f util/Dockerfile . - $runtime run -v $(pwd):${local_azpbs} $runtime_args $extra_args -ti azpbs_build:latest /bin/bash ${local_azpbs}/util/build.sh $local_scalelib + $runtime run --user "$(id -u):$(id -g)" -e HOME=/tmp -v "$(pwd):${local_azpbs}" $runtime_args $extra_args -ti azpbs_build:latest /bin/bash ${local_azpbs}/util/build.sh $local_scalelib } &> $log_file # Call the function to print the branch and the last commit From e50d944799e8ac026f87f76cc91c34cfb63d5da3 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Mon, 14 Sep 2026 17:09:33 +0000 Subject: [PATCH 08/18] Remove mount fix for now + rename export_name --- templates/openpbs.txt | 230 ++++++++++++------------------------------ 1 file changed, 63 insertions(+), 167 deletions(-) diff --git a/templates/openpbs.txt b/templates/openpbs.txt index 214fb21..5f2394d 100644 --- a/templates/openpbs.txt +++ b/templates/openpbs.txt @@ -17,10 +17,6 @@ Autoscale = $Autoscale Region = $Region KeyPairLocation = ~/.ssh/cyclecloud.pem Azure.Identities = $ManagedIdentity - - # Lustre mounts require termination notifications to unmount - EnableTerminateNotification = ${NFSType == "lustre" || NFSSchedType == "lustre" || AdditionalNFSType == "lustre" || EnableTerminateNotification} - TerminateNotificationTimeout = 10m [[[configuration]]] pbspro.version = $PBSVersion @@ -51,27 +47,25 @@ Autoscale = $Autoscale SSD = True [[[configuration cyclecloud.mounts.nfs_shared]]] - type = $NFSType + type = nfs mountpoint = /shared - export_path = ${ifThenElse(NFSType == "lustre", strcat("tcp:/lustrefs", NFSSharedExportPath), NFSSharedExportPath)} + export_path = $NFSSharedExportPath address = $NFSAddress options = $NFSSharedMountOptions - export_name = server + node_name = server [[[configuration cyclecloud.mounts.nfs_sched]]] - type = $NFSSchedType + type = nfs mountpoint = /sched - export_path = ${ifThenElse(NFSSchedType == "lustre", strcat("tcp:/lustrefs", NFSSchedExportPath), NFSSchedExportPath)} - address = ${ifThenElse(UseBuiltinSched, undefined, NFSSchedAddress)} - options = $NFSSchedMountOptions - export_name = server + disabled = $NFSSchedDisable + node_name = server [[[configuration cyclecloud.mounts.additional_nfs]]] - disabled = ${AdditionalNFS isnt true} - type = $AdditionalNFSType - address = $AdditionalNFSAddress + disabled = ${AdditionalNAS isnt true} + type = nfs + address = $AdditonalNFSAddress mountpoint = $AdditionalNFSMountPoint - export_path = ${ifThenElse(AdditionalNFSType == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath), AdditionalNFSExportPath)} + export_path = $AdditionalNFSExportPath options = $AdditionalNFSMountOptions @@ -83,8 +77,8 @@ Autoscale = $Autoscale [[[configuration]]] cyclecloud.discoverable=true - cyclecloud.mounts.nfs_sched.disabled = $UseBuiltinSched - cyclecloud.mounts.nfs_shared.disabled = $UseBuiltinShared + cyclecloud.mounts.nfs_sched.disabled = true + cyclecloud.mounts.nfs_shared.disabled = ${NFSType != "External"} pbspro.cron_method = $AzpbsCronMethod pbspro.queues.workq.ignore = ${Autoscale != true} pbspro.queues.htcq.ignore = ${Autoscale != true} @@ -96,38 +90,34 @@ Autoscale = $Autoscale AssociatePublicIpAddress = $UsePublicNetwork [[[volume sched]]] - Size = $SchedFilesystemSize + Size = 1024 SSD = True Mount = builtinsched - Persistent = True #TODO: does this need to be true - Disabled = ${!UseBuiltinSched} + Persistent = False [[[volume shared]]] - Size = $FilesystemSize + Size = ${ifThenElse(NFSType == "Builtin", FilesystemSize, 2)} SSD = True Mount = builtinshared - Persistent = True True #TODO: does this need to be true - Disabled = ${!UseBuiltinShared} #TODO: test new variable + Persistent = ${NFSType == "Builtin"} [[[configuration cyclecloud.mounts.builtinsched]]] - disabled = ${!UseBuiltinSched} mountpoint = /sched fs_type = xfs [[[configuration cyclecloud.mounts.builtinshared]]] - disabled = ${!UseBuiltinShared} + disabled = ${NFSType != "Builtin"} mountpoint = /shared fs_type = xfs [[[configuration cyclecloud.exports.builtinsched]]] - disabled = ${!UseBuiltinSched} export_path = /sched options = no_root_squash samba.enabled = false type = nfs [[[configuration cyclecloud.exports.builtinshared]]] - disabled = ${!UseBuiltinShared} + disabled = ${NFSType != "Builtin"} export_path = /shared samba.enabled = false type = nfs @@ -247,139 +237,61 @@ Order = 10 [parameters Network Attached Storage] Order = 15 - [[parameters Shared Storage]] - Order = 10 - - [[[parameter About Shared Storage]]] - HideLabel = true - Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template = '''

The directories /sched and /shared are network attached mounts and exist on all nodes of the cluster.
-
- Options for providing these mounts:
- [Builtin]: The server node is an NFS server that provides the mountpoint to the other nodes of the cluster.
- [External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server provides the mountpoint.
- [Azure Managed Lustre]: An Azure Managed Lustre deployment provides the mountpoint.
-

-

Note: the cluster must be terminated for changes to filesystem mounts to take effect.

''' - Conditions.Hidden := false [[parameters Scheduler Mount]] - Order = 20 - Label = File-system Mount for /sched - + Order = 5 [[[parameter About sched]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template = '''

OpenPBS configuration shared with cluster nodes is stored in the /sched directory. It is managed by the server node.

''' + Config.Template = '''

The directory /sched is a network attached mount and exists in all nodes of the cluster. + It's managed by the scheduler node. + To disable the mount of the /sched directory, and to supply your own for a hybrid scenario, select the checkbox below.''' Order = 6 - [[[parameter About sched part 2]]] - HideLabel = true - Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template = '''

Uncheck the box below to disable the built-in NFS export of the /sched directory and use an external file-system.

''' - Order = 7 - - [[[parameter UseBuiltinSched]]] - Label = Use Builtin NFS - Description = Use the builtin NFS for /sched - DefaultValue = true - ParameterType = Boolean - - [[[parameter NFSSchedDiskWarning]]] + [[[parameter NFSSchedDisable]]] HideLabel = true - Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the scheduler disk.

" - Conditions.Hidden := UseBuiltinSched - - [[[parameter NFSSchedType]]] - Label = FS Type - ParameterType = StringList - Config.Label = Type of shared filesystem to use for this cluster - Config.Plugin = pico.form.Dropdown - Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} - DefaultValue = nfs - Conditions.Hidden := UseBuiltinSched - - [[[parameter NFSSchedAddress]]] - Label = IP Address - Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. - Config.ParameterType = String - Conditions.Hidden := UseBuiltinSched - - [[[parameter NFSSchedExportPath]]] - Label = Export Path - Description = The path exported by the file system - DefaultValue = /sched - Conditions.Hidden := UseBuiltinSched - - [[[parameter NFSSchedMountOptions]]] - Label = Mount Options - Description = File system client mount options - Conditions.Hidden := UseBuiltinSched - - [[[parameter SchedFilesystemSize]]] - Label = Size (GB) - Description = The filesystem size (cannot be changed after initial start) - DefaultValue = 1024 - Config.Plugin = pico.form.NumberTextBox - Config.MinValue = 10 - Config.MaxValue = 10240 - Config.IntegerOnly = true - Conditions.Excluded := !UseBuiltinSched + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = External Scheduler [[parameters Default NFS Share]] - Order = 30 - Label = File-system Mount for /shared - + Order = 10 [[[parameter About shared]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template = '''

Users' home directories reside within the /shared mountpoint with the base homedir /shared/home.

''' - Order = 6 - - [[[parameter About shared part 2]]] - HideLabel = true - Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template = '''

Uncheck the box below to disable the built-in NFS export of the /shared directory and use an external file-system.

''' - Order = 7 + Config.Template := "

The directory /shared is a network attached mount and exists in all nodes of the cluster. Users' home directories reside within this mountpoint with the base homedir /shared/home.

There are two options for providing this mount:
[Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster.
[External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server, provides the mountpoint.

Note: the cluster must be terminated for this to take effect.

" + Order = 20 - [[[parameter UseBuiltinShared]]] - Label = Use Builtin NFS - Description = Use the builtin NFS for /shared - DefaultValue = true - ParameterType = Boolean + [[[parameter NFSType]]] + Label = NFS Type + ParameterType = StringList + Config.Label = Type of NFS to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="External"], [Label="Builtin"; Value="Builtin"]} + DefaultValue = Builtin [[[parameter NFSDiskWarning]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" - Conditions.Hidden := UseBuiltinShared - - [[[parameter NFSType]]] - Label = FS Type - ParameterType = StringList - Config.Label = Type of filesystem to use for /shared - Config.Plugin = pico.form.Dropdown - Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} - DefaultValue = nfs - Conditions.Hidden := UseBuiltinShared + Config.Template := "

Warning: switching an active cluster over to NFS will delete the shared disk.

" + Conditions.Hidden := NFSType != "External" [[[parameter NFSAddress]]] - Label = IP Address - Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Label = NFS IP Address + Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String - Conditions.Hidden := UseBuiltinShared + Conditions.Hidden := NFSType != "External" [[[parameter NFSSharedExportPath]]] - Label = Export Path + Label = Shared Export Path Description = The path exported by the file system DefaultValue = /shared - Conditions.Hidden := UseBuiltinShared + Conditions.Hidden := NFSType != "External" [[[parameter NFSSharedMountOptions]]] - Label = Mount Options - Description = File system client mount options - Conditions.Hidden := UseBuiltinShared + Label = NFS Mount Options + Description = NFS Client Mount Options + Conditions.Hidden := NFSType != "External" [[[parameter FilesystemSize]]] Label = Size (GB) @@ -390,56 +302,44 @@ Order = 15 Config.MinValue = 10 Config.MaxValue = 10240 Config.IntegerOnly = true - Conditions.Excluded := !UseBuiltinShared + Conditions.Excluded := NFSType != "Builtin" [[parameters Additional NFS Mount]] - Order = 40 - Label = Additional Filesystem Mount - - [[[parameter Additional Shared FS Mount Readme]]] + Order = 20 + [[[parameter Additional NFS Mount Readme]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Mount another shared file-system endpoint on the cluster nodes.

" + Config.Template := "

Mount another NFS endpoint on the cluster nodes.

" Order = 20 - [[[parameter AdditionalNFS]]] + [[[parameter AdditionalNAS]]] HideLabel = true DefaultValue = false Widget.Plugin = pico.form.BooleanCheckBox - Widget.Label = Add Shared Filesystem mount - - [[[parameter AdditionalNFSType]]] - Label = FS Type - ParameterType = StringList - Config.Label = Shared filesystem type of the additional mount - Config.Plugin = pico.form.Dropdown + Widget.Label = Add NFS mount - Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} - DefaultValue = nfs - Conditions.Excluded := AdditionalNFS isnt true - - [[[parameter AdditionalNFSAddress]]] - Label = IP Address - Description = The IP address or hostname of the additional mount. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + [[[parameter AdditonalNFSAddress]]] + Label = NFS IP Address + Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String - Conditions.Excluded := AdditionalNFS isnt true + Conditions.Excluded := AdditionalNAS isnt true [[[parameter AdditionalNFSMountPoint]]] - Label = Mount Point + Label = NFS Mount Point Description = The path at which to mount the Filesystem DefaultValue = /data - Conditions.Excluded := AdditionalNFS isnt true + Conditions.Excluded := AdditionalNAS isnt true [[[parameter AdditionalNFSExportPath]]] - Label = Export Path + Label = NFS Export Path Description = The path exported by the file system DefaultValue = /data - Conditions.Excluded := AdditionalNFS isnt true + Conditions.Excluded := AdditionalNAS isnt true [[[parameter AdditionalNFSMountOptions]]] - Label = Mount Options - Description = Filesystem Client Mount Options - Conditions.Excluded := AdditionalNFS isnt true + Label = NFS Mount Options + Description = NFS Client Mount Options + Conditions.Excluded := AdditionalNAS isnt true @@ -469,10 +369,6 @@ Order = 20 Config.Increment = 64 DefaultValue = 0 - [[[parameter EnableTerminateNotification]]] - Label = Enable Termination notifications - DefaultValue = false - [[parameters Software]] From be2d7a73cdc3ea9eac2660ae6ebc1caef52fcb31 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Mon, 14 Sep 2026 19:23:19 +0000 Subject: [PATCH 09/18] Use mount address for node search and move pbspro install --- specs/default/cluster-init/files/utils.sh | 31 ------------------- specs/default/cluster-init/install-hook.sh | 27 +++++++++++----- .../cluster-init/roles/execute-install.sh | 12 +++---- .../cluster-init/roles/login-install.sh | 9 ++---- .../cluster-init/roles/server-install.sh | 15 +-------- 5 files changed, 28 insertions(+), 66 deletions(-) diff --git a/specs/default/cluster-init/files/utils.sh b/specs/default/cluster-init/files/utils.sh index de46cce..9a0f574 100644 --- a/specs/default/cluster-init/files/utils.sh +++ b/specs/default/cluster-init/files/utils.sh @@ -36,34 +36,3 @@ function get_package_name() { echo "$package_name" fi } - -function get_server_hostname() { - local server_hostname=$(jetpack config pbspro.scheduler "") || fail - local cluster_name=$(jq -r .cluster "$CONFIG_PATH") || fail - - if [[ -z "$server_hostname" ]]; then - local -r max_retries=10 - local -r retry_delay=15 - local attempt=1 - - # Read server hostname from azpbs.env config file generated by server node - if [[ ! -e "/sched/${cluster_name}/azpbs.env" ]]; then - while [[ $attempt -lt $max_retries ]]; do - sleep $retry_delay - ((attempt+=1)) - - if [[ -e "/sched/${cluster_name}/azpbs.env" ]]; then - break; - fi - done - - if [[ $attempt == $max_retries ]]; then - fail "Failed to read /sched/${cluster_name}/azpbs.env after $max_retries attempts. Exiting." - fi - fi - source "/sched/${cluster_name}/azpbs.env" || fail - echo "$PBS_SCHEDULER_HOSTNAME" - else - echo "$server_hostname" - fi -} \ No newline at end of file diff --git a/specs/default/cluster-init/install-hook.sh b/specs/default/cluster-init/install-hook.sh index 7df6e49..f7cb00e 100644 --- a/specs/default/cluster-init/install-hook.sh +++ b/specs/default/cluster-init/install-hook.sh @@ -1,15 +1,28 @@ #!/bin/bash source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -bash "${CYCLECLOUD_PROJECT_PATH}/default/files/hwlocs-install.sh" || fail - ROLE=$(jetpack config pbspro.role "") || fail -echo "jetpack config pbspro.role $ROLE" - case "$ROLE" in - server) bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/server-install.sh" || fail ;; - login) bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/login-install.sh" || fail ;; - execute) bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/execute-install.sh" || fail ;; + server) + PACKAGE_TYPE="server" + ROLE_SCRIPT="server-install.sh" + ;; + login) + PACKAGE_TYPE="client" + ROLE_SCRIPT="login-install.sh" + ;; + execute) + PACKAGE_TYPE="execution" + ROLE_SCRIPT="execute-install.sh" + ;; *) fail "Unknown pbspro.role '$ROLE'" ;; esac + +bash "${CYCLECLOUD_PROJECT_PATH}/default/files/hwlocs-install.sh" || fail + +PACKAGE_NAME=$(get_package_name "$PACKAGE_TYPE") || fail +jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail +yum install -y -q "/tmp/$PACKAGE_NAME" || fail + +bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/${ROLE_SCRIPT}" || fail diff --git a/specs/default/cluster-init/roles/execute-install.sh b/specs/default/cluster-init/roles/execute-install.sh index afe34b1..02b94a8 100755 --- a/specs/default/cluster-init/roles/execute-install.sh +++ b/specs/default/cluster-init/roles/execute-install.sh @@ -3,24 +3,20 @@ source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 EXECUTE_HOSTNAME=$(jetpack config hostname) || fail -PACKAGE_NAME=$(get_package_name "execution") || fail -SERVER_HOSTNAME=$(get_server_hostname) || fail +SERVER_IP_ADDRESS=$(jetpack config cyclecloud.mounts.nfs_sched.address "") || fail # Forces execute node's hostname to be updated (scalelib is blocked until the hostname is correct) # TODO: this installation status should be done by jetpack before cluster-inits are run "${CYCLECLOUD_HOME}/system/embedded/bin/python" -c "import jetpack.converge as jc; jc._send_installation_status('warning')" -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail - -if [[ -n "$SERVER_HOSTNAME" ]]; then - echo "$SERVER_HOSTNAME" > /var/spool/pbs/server_name +if [[ -n "$SERVER_IP_ADDRESS" ]]; then + echo "$SERVER_IP_ADDRESS" > /var/spool/pbs/server_name chmod 0644 /var/spool/pbs/server_name || fail cp "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/mom_config.template" /var/spool/pbs/mom_priv/config || fail chmod 0644 /var/spool/pbs/mom_priv/config || fail - sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ + sed -e "s|__SERVERNAME__|${SERVER_IP_ADDRESS}|g" \ "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail chmod 0644 /etc/pbs.conf || fail fi diff --git a/specs/default/cluster-init/roles/login-install.sh b/specs/default/cluster-init/roles/login-install.sh index 1379a65..8f31146 100755 --- a/specs/default/cluster-init/roles/login-install.sh +++ b/specs/default/cluster-init/roles/login-install.sh @@ -2,14 +2,11 @@ source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -PACKAGE_NAME=$(get_package_name "client") || fail -SERVER_HOSTNAME=$(get_server_hostname) || fail +SERVER_IP_ADDRESS=$(jetpack config cyclecloud.mounts.nfs_sched.address "") || fail -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail -if [[ -n "$SERVER_HOSTNAME" ]]; then - sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ +if [[ -n "$SERVER_IP_ADDRESS" ]]; then + sed -e "s|__SERVERNAME__|${SERVER_IP_ADDRESS}|g" \ "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail chmod 0644 /etc/pbs.conf || fail fi diff --git a/specs/default/cluster-init/roles/server-install.sh b/specs/default/cluster-init/roles/server-install.sh index 3544e48..5846ac6 100644 --- a/specs/default/cluster-init/roles/server-install.sh +++ b/specs/default/cluster-init/roles/server-install.sh @@ -3,7 +3,6 @@ source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail echo "hi im the server node" -PACKAGE_NAME=$(get_package_name "server") || fail CLUSTER_NAME=$(jq -r .cluster "$CONFIG_PATH") || fail CONNECTION_URL=$(jq -r .url "$CONFIG_PATH") || fail IGNORE_WORKQ=$(jetpack config pbspro.queues.workq.ignore "False") || fail @@ -12,18 +11,6 @@ CRON_METHOD=$(jetpack config pbspro.cron_method "pbs_cron") || fail PBSPRO_AUTOSCALE_PROJECT_HOME="/opt/cycle/pbspro" PBSPRO_AUTOSCALE_INSTALLER="cyclecloud-pbspro-pkg-${PBSPRO_AUTOSCALE_VERSION}.tar.gz" -mkdir -p "/sched/${CLUSTER_NAME}" || fail - -cat << EOF > "/sched/${CLUSTER_NAME}/azpbs.env" -#!/bin/bash -PBS_SCHEDULER_HOSTNAME=$(hostname) -PBS_SCHEDULER_IP=$(hostname -i) - -EOF -chmod a+r "/sched/${CLUSTER_NAME}/azpbs.env" || fail - -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail mkdir -p -m 0755 "$PBSPRO_AUTOSCALE_PROJECT_HOME" || fail @@ -68,7 +55,7 @@ fi ./initialize_default_queues.sh || fail ./install.sh --install-python3 --venv "${INSTALLDIR}/venv" --cron-method "$CRON_METHOD" || fail - +#TODO: modify readme to get this ./generate_autoscale_json.sh --install-dir "$INSTALLDIR" \ --url "$CONNECTION_URL" \ --cluster-name "$CLUSTER_NAME" \ From bbbcda69325e09f6583c1a7a1c385146de9f0529 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Mon, 14 Sep 2026 19:23:50 +0000 Subject: [PATCH 10/18] Delete old scripts --- specs/execute/cluster-init/stages/activate.sh | 3 - specs/execute/cluster-init/stages/install.sh | 70 --------------- specs/login/cluster-init/stages/install.sh | 20 ----- specs/server/cluster-init/stages/install.sh | 85 ------------------- 4 files changed, 178 deletions(-) delete mode 100755 specs/execute/cluster-init/stages/activate.sh delete mode 100755 specs/execute/cluster-init/stages/install.sh delete mode 100755 specs/login/cluster-init/stages/install.sh delete mode 100644 specs/server/cluster-init/stages/install.sh diff --git a/specs/execute/cluster-init/stages/activate.sh b/specs/execute/cluster-init/stages/activate.sh deleted file mode 100755 index ba338ca..0000000 --- a/specs/execute/cluster-init/stages/activate.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -systemctl start pbs || exit 1 \ No newline at end of file diff --git a/specs/execute/cluster-init/stages/install.sh b/specs/execute/cluster-init/stages/install.sh deleted file mode 100755 index 7a501ba..0000000 --- a/specs/execute/cluster-init/stages/install.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail - -"${CYCLECLOUD_PROJECT_PATH}/default/scripts/hwlocs-install.sh" || fail - -EXECUTE_HOSTNAME=$(jetpack config hostname) || fail -PACKAGE_NAME=$(get_package_name "execution") || fail -SERVER_HOSTNAME=$(get_server_hostname) || fail - -# Forces execute node's hostname to be updated (scalelib is blocked until the hostname is correct) -# TODO: this installation status should be done by jetpack before cluster-inits are run -"${CYCLECLOUD_HOME}/system/embedded/bin/python" -c "import jetpack.converge as jc; jc._send_installation_status('warning')" - -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail - -if [[ -n "$SERVER_HOSTNAME" ]]; then - echo "$SERVER_HOSTNAME" > /var/spool/pbs/server_name - chmod 0644 /var/spool/pbs/server_name || fail - - cp "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/mom_config.template" /var/spool/pbs/mom_priv/config || fail - chmod 0644 /var/spool/pbs/mom_priv/config || fail - - sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ - "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail - chmod 0644 /etc/pbs.conf || fail -fi - -await_node_definition() { - if ! /opt/pbs/bin/pbsnodes "$EXECUTE_HOSTNAME"; then - echo "${EXECUTE_HOSTNAME} is not in the cluster yet. Retrying next converge" 1>&2 - return 1 - fi -} - -readonly MAX_RETRIES=10 -readonly RETRY_DELAY=15 -ATTEMPT=1 -if ! await_node_definition; then - while [[ $ATTEMPT -lt $MAX_RETRIES ]]; do - sleep $RETRY_DELAY - ((ATTEMPT+=1)) - - if await_node_definition; then - break; - fi - done - - if [[ $ATTEMPT == $MAX_RETRIES ]]; then - fail "Command failed after $MAX_RETRIES attempts. Exiting." - fi -fi - -# This block will execute only if the "execute" node is defined in the PBS server -NODE_CREATED_GUARD="pbs.nodecreated" -if [[ -f "$NODE_CREATED_GUARD" ]]; then - echo "Node has already been created, skipping joining checks" -else - NODE_ATTRS=$(/opt/pbs/bin/pbsnodes "$EXECUTE_HOSTNAME") || fail - NODE_ID=$(jetpack config cyclecloud.node.id) || fail - if ! echo "$NODE_ATTRS" | bool grep -qi "$NODE_ID"; then - fail "Stale entry found for $EXECUTE_HOSTNAME. Waiting for autoscaler to update this before joining." - fi - - /opt/pbs/bin/pbsnodes -o "$EXECUTE_HOSTNAME" -C 'cyclecloud offline' || fail - - touch "$NODE_CREATED_GUARD" || fail -fi \ No newline at end of file diff --git a/specs/login/cluster-init/stages/install.sh b/specs/login/cluster-init/stages/install.sh deleted file mode 100755 index 9840895..0000000 --- a/specs/login/cluster-init/stages/install.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail - -"${CYCLECLOUD_PROJECT_PATH}/default/scripts/hwlocs-install.sh" || fail - -PACKAGE_NAME=$(get_package_name "client") || fail -SERVER_HOSTNAME=$(get_server_hostname) || fail - -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail - -if [[ -n "$SERVER_HOSTNAME" ]]; then - sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ - "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail - chmod 0644 /etc/pbs.conf || fail -fi - -/opt/pbs/bin/qmgr -c "set server flatuid=true" || fail \ No newline at end of file diff --git a/specs/server/cluster-init/stages/install.sh b/specs/server/cluster-init/stages/install.sh deleted file mode 100644 index 1b45e92..0000000 --- a/specs/server/cluster-init/stages/install.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail - -"${CYCLECLOUD_PROJECT_PATH}/default/scripts/hwlocs-install.sh" || fail - -PACKAGE_NAME=$(get_package_name "server") || fail -CLUSTER_NAME=$(jq -r .cluster "$CONFIG_PATH") || fail -CONNECTION_URL=$(jq -r .url "$CONFIG_PATH") || fail -IGNORE_WORKQ=$(jetpack config pbspro.queues.workq.ignore "False") || fail -IGNORE_HTCQ=$(jetpack config pbspro.queues.htcq.ignore "False") || fail -CRON_METHOD=$(jetpack config pbspro.cron_method "pbs_cron") || fail -PBSPRO_AUTOSCALE_PROJECT_HOME="/opt/cycle/pbspro" -PBSPRO_AUTOSCALE_INSTALLER="cyclecloud-pbspro-pkg-${PBSPRO_AUTOSCALE_VERSION}.tar.gz" - -mkdir -p "/sched/${CLUSTER_NAME}" || fail - -cat << EOF > "/sched/${CLUSTER_NAME}/azpbs.env" -#!/bin/bash -PBS_SCHEDULER_HOSTNAME=$(hostname) -PBS_SCHEDULER_IP=$(hostname -i) - -EOF -chmod a+r "/sched/${CLUSTER_NAME}/azpbs.env" || fail - -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail - -mkdir -p -m 0755 "$PBSPRO_AUTOSCALE_PROJECT_HOME" || fail - -mkdir -p -m 750 /var/spool/pbs/sched_priv || fail - -cp "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/sched.config" /var/spool/pbs/sched_priv/sched_config || fail -chmod 0644 /var/spool/pbs/sched_priv/sched_config || fail - -systemctl enable --now pbs || fail - -source /etc/profile.d/pbs.sh || fail -PATH=$PATH:/root/bin - -cd "$BOOTSTRAP_HOME" || fail # TODO: find a new location instead of BOOTSTRAP_HOME - -rm -f "$PBSPRO_AUTOSCALE_INSTALLER" || fail - -jetpack download "$PBSPRO_AUTOSCALE_INSTALLER" --project pbspro ./ || fail - -if [ -e cyclecloud-pbspro ]; then - rm -rf cyclecloud-pbspro/ || fail -fi - -tar xzf "$PBSPRO_AUTOSCALE_INSTALLER" || fail - -cd cyclecloud-pbspro/ || fail - -INSTALLDIR=$(realpath "$PBSPRO_AUTOSCALE_PROJECT_HOME") || fail -mkdir -p "${INSTALLDIR}/venv" || fail - -IGNORE_QUEUES_ARG="" -if [[ "$IGNORE_WORKQ" == "True" && "$IGNORE_HTCQ" == "True" ]]; then - IGNORE_QUEUES_ARG="--ignore-queues workq,htcq" -elif [[ "$IGNORE_WORKQ" == "True" ]]; then - IGNORE_QUEUES_ARG="--ignore-queues workq" -elif [[ "$IGNORE_HTCQ" == "True" ]]; then - IGNORE_QUEUES_ARG="--ignore-queues htcq" -fi - -./initialize_pbs.sh || fail - -./initialize_default_queues.sh || fail - -./install.sh --install-python3 --venv "${INSTALLDIR}/venv" --cron-method "$CRON_METHOD" || fail - -./generate_autoscale_json.sh --install-dir "$INSTALLDIR" \ - --username "$(jetpack config cyclecloud.config.username)" \ - --password "$(jetpack config cyclecloud.config.password)" \ - --url "$CONNECTION_URL" \ - --cluster-name "$CLUSTER_NAME" \ - $IGNORE_QUEUES_ARG \ - || fail - -ls "${PBSPRO_AUTOSCALE_PROJECT_HOME}/autoscale.json" || fail -azpbs connect || fail - -systemctl restart pbs || fail \ No newline at end of file From 88864bebdaf1b4b726d2c62c5ddcc04b326cf69e Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Mon, 14 Sep 2026 19:52:54 +0000 Subject: [PATCH 11/18] Remove directing users to insert jetpack username and password into generate_autoscale_json.sh --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 1dac053..d8f6a17 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,6 @@ cd cyclecloud-pbspro # If you have jetpack available, you may use the following: # ./generate_autoscale_json.sh --install-dir /opt/cycle/pbspro \ -# --username $(jetpack config cyclecloud.config.username) \ -# --password $(jetpack config cyclecloud.config.password) \ # --url $(jetpack config cyclecloud.config.web_server) \ # --cluster-name $(jetpack config cyclecloud.cluster.name) From 23804c1404117da60a179ee84cbffeff56a372a9 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Mon, 14 Sep 2026 19:57:10 +0000 Subject: [PATCH 12/18] Remove tests --- specs/default/cluster-init/tests/helper.py | 82 -- .../cluster-init/tests/test_execute.py | 22 - .../default/cluster-init/tests/test_submit.py | 175 ---- specs/default/cluster-init/tests/tryme.py | 868 ------------------ 4 files changed, 1147 deletions(-) delete mode 100644 specs/default/cluster-init/tests/helper.py delete mode 100644 specs/default/cluster-init/tests/test_execute.py delete mode 100644 specs/default/cluster-init/tests/test_submit.py delete mode 100644 specs/default/cluster-init/tests/tryme.py diff --git a/specs/default/cluster-init/tests/helper.py b/specs/default/cluster-init/tests/helper.py deleted file mode 100644 index 28a0f97..0000000 --- a/specs/default/cluster-init/tests/helper.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -from jetpack import config -import subprocess -import os -import sys -import traceback -from io import StringIO -import logging -import platform - -if platform.system() != 'Windows': - import pwd - - -logger = logging.getLogger(__name__) - - -def get_user_profile(user_name, cwd=None): - '''Get user id, group id, and environment variables for specified user''' - pw_record = pwd.getpwnam(user_name) - user_name = pw_record.pw_name - user_home_dir = pw_record.pw_dir - user_uid = pw_record.pw_uid - user_gid = pw_record.pw_gid - env = os.environ.copy() - env['HOME'] = user_home_dir - env['LOGNAME'] = user_name - env['PWD'] = cwd or user_home_dir - env['USER'] = user_name - return user_uid, user_gid, env - - -def demote_to_user(user_uid, user_gid): - '''Demote current process to given user and group''' - def result(): - os.setgid(user_gid) - os.setuid(user_uid) - return result - - -def sudo_check_call(cmd_args, username, cwd=None, env=None): - uid, gid, user_env = get_user_profile(username, cwd=cwd) - user_env.update(env) - subprocess.check_call(cmd_args, cwd=cwd, env=env, preexec_fn=demote_to_user(uid, gid)) - return True - - -def sudo_check_output(cmd_args, username, cwd=None, env={}): - uid, gid, user_env = get_user_profile(username, cwd=cwd) - user_env.update(env) - return subprocess.check_output(cmd_args, cwd=cwd, env=user_env, preexec_fn=demote_to_user(uid, gid)) - - -def get_chef_role(role_name): - return role_name in config.get('roles', []) - - -def exception_to_str(): - t, e, tb = sys.exc_info() - f = StringIO() - traceback.print_tb(tb, None, f) - stack_trace = f.getvalue() - if e.message == '': - exception_message = getattr(e, 'strerror', '') - else: - exception_message = e.message - message = "Encountered exception of type %s with error message:" % type(e) - - if exception_message != '': - message = message + '\n' + exception_message - message = message + '\n' + 'Stacktrace:\n%s' % stack_trace - return message - - -def count_nodes_by_os(output, operating_system): - nodes = output.strip().split('\n\n') - if len(nodes) < 1: - return 0 - opsys = operating_system.upper() - return len(filter(lambda node: 'OpSys = "%s"' % opsys in node, nodes)) diff --git a/specs/default/cluster-init/tests/test_execute.py b/specs/default/cluster-init/tests/test_execute.py deleted file mode 100644 index dabdd23..0000000 --- a/specs/default/cluster-init/tests/test_execute.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -import unittest -import subprocess -import jetpack - -PBSPRO_ROLE = jetpack.config.get("pbspro.role", None) - -@unittest.skipUnless(PBSPRO_ROLE == "execute" or PBSPRO_ROLE == "login", "execute or login-only test") -class TestExecute(unittest.TestCase): - - def test_simple(self): - p = subprocess.Popen(['/opt/pbs/bin/qstat'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - if hasattr(stdout, "decode"): - stdout = stdout.decode() - stderr = stderr.decode() - - self.assertEqual(0, p.returncode, msg="Call to qstat failed with Stderr: %s\nStdout%s" - % (stderr, stdout)) - diff --git a/specs/default/cluster-init/tests/test_submit.py b/specs/default/cluster-init/tests/test_submit.py deleted file mode 100644 index 4d408b3..0000000 --- a/specs/default/cluster-init/tests/test_submit.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -import json -import logging -import os -from subprocess import check_output, check_call, CalledProcessError -import time -import unittest -import uuid - -import helper -import jetpack -from jetpack import props - - -jetpack.util.setup_logging() - -logger = logging.getLogger() - -CLUSTER_USER = props.get("cyclecloud.owner") -PBSPRO_ROLE = jetpack.config.get("pbspro.role", None) - - -def readfile_if_exist(filename): - if not os.path.exists(filename): - return '' - - with open(filename) as f: - content = f.read() - return content - - -def write_job_script(): - uid, gid, _ = helper.get_user_profile(CLUSTER_USER) - job_script = '/shared/home/%s/run_hostname.sh' % CLUSTER_USER - - with open(job_script, 'w') as f: - f.write(''' -#!/bin/bash -e - -cat $PBS_NODEFILE > $1 -sleep 3600 -'''.strip()) - - os.chown(job_script, uid, gid) - os.chmod(job_script, 0x0755) - - return job_script - -@unittest.skipUnless(PBSPRO_ROLE == "server", "server-only test") -class TestSubmit(unittest.TestCase): - - def setUp(self): - self.userhome = "/shared/home/" + CLUSTER_USER - self.results_dir = os.path.join(self.userhome, "test_submit") - if not os.path.exists(self.results_dir): - uid, gid, _ = helper.get_user_profile(CLUSTER_USER) - os.makedirs(self.results_dir) - os.chown(self.results_dir, uid, gid) - - def test_submissions(self): - if jetpack.config.get("pbspro.skip_integration_tests", True): - return - - job_script = write_job_script() - - def submit_job(prefix, qsub_args): - job_uuid = prefix + str(uuid.uuid4()) - output_file_base = self.userhome + '/test_submit/%s' % job_uuid - - helper.sudo_check_output(['/opt/pbs/bin/qsub'] + qsub_args + ["-N", job_uuid, "--", job_script, output_file_base], - CLUSTER_USER, cwd=self.userhome) - return job_uuid - - simple = submit_job("simple-", []) - simple_htcq = submit_job("simple_htcq-", ["-q", "htcq"]) - multipart_select = submit_job("multipart_select-", ["-l", "select=2:ncpus=2+1:ncpus=2"]) - vscatter_excl = submit_job("vscatter_excl=", ["-l", "select=2:ncpus=1", "-l", "place=vscatter:excl"]) - smp = submit_job("smp-", ["-l", "select=2:ncpus=1", "-l", "place=pack:group=host"]) - old_nodes = submit_job("old_nodes-", ["-l", "nodes=2:ppn=2"]) - - # timer that releases the held jobs is set to 15 seconds - time.sleep(30) - - jobs = json.loads(check_output(['qstat', '-f', '-F', 'json'])) - jobs_by_name = {} - for job_id, job in list(jobs.get("Jobs", {}).items()): - job["Job_Id"] = job_id - jobs_by_name[job["Job_Name"]] = job - - def check_job(job_uuid, select, place, slot_type="execute"): - self.assertEqual(select, jobs_by_name[job_uuid]["Resource_List"]["select"]) - self.assertEqual(place, jobs_by_name[job_uuid]["Resource_List"]["place"]) - self.assertEqual(slot_type, jobs_by_name[job_uuid]["Resource_List"]["slot_type"]) - - check_job(simple, "1:ncpus=1:slot_type=execute:ungrouped=false", "pack:group=group_id") - check_job(simple_htcq, "1:ncpus=1:slot_type=execute:ungrouped=true", "pack") - check_job(multipart_select, "2:ncpus=2+1:ncpus=2", "scatter:group=group_id") - check_job(vscatter_excl, "2:ncpus=1", "vscatter:excl:group=group_id") - check_job(smp, "2:ncpus=1", "pack:group=host") - check_job(old_nodes, "2:ncpus=2:mpiprocs=2", "scatter:group=group_id") - - hosts = {} - - def wait_for_results(): - omega = time.time() + 1200 - at_least_one_missing = True - - while at_least_one_missing and time.time() < omega: - try: - # continually collect host information. Some may get shutdown while we are waiting for other jobs to complete. - nodes = json.loads(check_output(["pbsnodes", "-a", "-F", "json"])) - for hostname, node in list(nodes["nodes"].items()): - hosts[hostname.lower()] = node - except CalledProcessError: - # pbsnodes exits with 1 if there are no nodes, just ignore. - pass - - at_least_one_missing = False - result = {} - for job_id in [simple, simple_htcq, multipart_select, vscatter_excl, smp, old_nodes]: - path = os.path.join(self.results_dir, job_id) - if not os.path.exists(path): - at_least_one_missing = True - logger.warn("%s does not exist yet." % path) - else: - with open(path) as fr: - result[job_id] = result.get(job_id, []) - for line in fr.read().splitlines(): - hostname_short = line.strip().split(".")[0].lower() - if hostname_short: - result[job_id].append(hostname_short) - - time.sleep(5) - - return result - - result = wait_for_results() - - def check_hosts(job_uuid, expected_procs, expected_hosts, placed): - self.assertIn(job_uuid, result, "Job %s did not complete" % job_uuid) - self.assertEqual(expected_procs, len(result[job_uuid])) - - # for some types of jobs, there is no guarantee on how many hosts they will land on. - if expected_hosts > 0: - self.assertEqual(expected_hosts, len(set(result[job_uuid]))) - - for hostname in result[job_uuid]: - hostname = hostname.lower() - if placed: - self.assertIsNotNone(hosts[hostname]["resources_available"].get("group_id")) - else: - self.assertIsNone(hosts[hostname]["resources_available"].get("group_id")) - #self.assertEquals(str(not placed).lower(), str(hosts[hostname]["resources_available"]["ungrouped"]).lower()) - - check_hosts(simple, 1, 1, True) - check_hosts(simple_htcq, 1, 1, False) - # -l place=scatter means best effort to scatter, but the jobs could land on a single machine. - check_hosts(multipart_select, 3, -1, True) - check_hosts(vscatter_excl, 2, 2, True) - check_hosts(smp, 2, 1, True) - # 4 mpi procs means 4 processes, so we will see 4 entries in the list but only 2 machines - check_hosts(old_nodes, 4, 2, True) - - # now terminate the sleep jobs so that scale down can happen. - # note, this ensures we scaled at least as many nodes as required. - # if a node was scaled up that wasn't needed, it will fail the scale down assertions in labrat - for job_name in [simple, simple_htcq, multipart_select, vscatter_excl, smp, old_nodes]: - job_id = jobs_by_name[job_name]["Job_Id"] - check_call(["qdel", job_id]) - - -if __name__ == "__main__": - unittest.main() diff --git a/specs/default/cluster-init/tests/tryme.py b/specs/default/cluster-init/tests/tryme.py deleted file mode 100644 index cfd0d0e..0000000 --- a/specs/default/cluster-init/tests/tryme.py +++ /dev/null @@ -1,868 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Bryan W. Berry -# License: BSD New, see LICENSE for details. - -import sys -import os -import time -from abc import ABCMeta, abstractmethod -from functools import wraps, total_ordering - -# Is this Python 3? -PY3 = sys.version_info > (3, 0) - - -class Null(object): - """Null represents nothing.""" - # pylint: disable = too-few-public-methods - def __repr__(self): - return 'Null' - - def __call__(self): - return self - - -# pylint: disable = invalid-name -#: The Null object. -Null = Null() - - -class Unit(object): - """Descriptor that always return the owner monad, used for ``unit``.""" - # pylint: disable = too-few-public-methods - def __get__(self, instance, cls): - """Returns the owner monad.""" - return cls - - -@total_ordering -class Ord(object): - """Mixin class that implements rich comparison ordering methods.""" - # pylint: disable = too-few-public-methods - def __eq__(self, other): - if self is other: - return True - elif not isinstance(other, type(self)): - return NotImplemented - else: - return self._value == other._value - - def __lt__(self, other): - if self is other: - return False - elif isinstance(other, type(self)): - return self._value < other._value - else: - fmt = "unorderable types: {} and {}'".format - raise TypeError(fmt(type(self), type(other))) - - -class Monad(object): - """The Monad Class. - - This is just a base class - """ - __metaclass__ = ABCMeta - - def __init__(self, value): - self._value = value - - def __repr__(self): - return '{cls}({value})'.format( - cls=type(self).__name__, value=repr(self._value)) - - @abstractmethod - def map(self, function): - """The map operation. - - ``function`` is a function that maps from the underlying value to a - monadic type, something like signature ``f :: a -> M a`` in haskell's - term. - """ - return NotImplemented - - #: The ``unit`` of monad. - unit = Unit() - - -class NoSuchElementError(Exception): - pass - - -class FailureError(Exception): - pass - - -class InvalidTryError(Exception): - pass - - -class Try(Monad, Ord): - """A wrapper for operations that may fail - - Represents values/computations with two possibilities. - - :param value: value to contain - :param message: (optional) message to output to console if to_console is called. - If None, a string representation of the contained value is output. - Defaults to ``None`` - :param start: (optional) start time for the operation, in seconds since the UNIX epoch - `time.time()` is typically used for this value. Defaults to ``None``. - :type start: int - :param end: (optional) end time for the operation, in seconds since the UNIX epoch - ``time.time()`` is typically used for this value. Defaults to ``None``. - :type end: int - :param count: (optional) number of times the operation has been executed. Defaults to ``1`` - :type end: int - - Usage:: - - >>> Success(42) - Success(42) - >>> Success([1, 2, 3]) - Success([1, 2, 3]) - >>> Failure('Error') - Failure('Error') - >>> Success(Failure('Error')) - Success(Failure('Error')) - >>> isinstance(Success(1), Try) - True - >>> isinstance(Failure(None), Try) - True - >>> saving = 100 - >>> insolvent = Failure('I am insolvent') - >>> spend = lambda cost: insolvent if cost > saving else Success(saving - cost) - >>> spend(90) - Success(10) - >>> spend(120) - Failure('I am insolvent') - - Map operation with ``map``, applies function to value only if it is a Success and returns a Success - - >>> inc = lambda n: n + 1 - >>> Success(0) - Success(0) - >>> Success(0).map(inc) - Success(1) - >>> Success(0).map(inc).map(inc) - Success(2) - >>> Failure(0).map(inc) - Failure(0) - - Comparison with ``==``, as long as they are the same type and what's - wrapped inside are comparable. - - >>> Failure(42) == Failure(42) - True - >>> Success(42) == Success(42) - True - >>> Failure(42) == Success(42) - False - - A :py:class:`Failure` is less than a :py:class:`Success`, or compare the two by - the values inside if thay are of the same type. - - >>> Failure(42) < Success(42) - True - >>> Success(0) > Failure(100) - True - >>> Failure('Error message') > Success(42) - False - >>> Failure(100) > Failure(42) - True - >>> Success(-2) < Success(-1) - True - """ - def __init__(self, value, message=None, start=None, end=None, count=1): - super(Try, self).__init__(value) - self._message = message - self._start = start - self._end = end - self._count = count - if (start is None and end is not None) or (end is None and start is not None): - raise InvalidTryError( - "The start and end argument must either be both None or not None") - - if type(self) is Try: - raise NotImplementedError('Please use Failure or Success instead') - - def map(self, function): - """The map operation of :py:class:`Try` to Success instances - - Applies function to the value if and only if this is a - :py:class:`Success`. - """ - constructor = type(self) - if self.succeeded(): - return constructor(function(self._value)) - else: - return self - - def map_failure(self, function): - """The map operation of :py:class:`Try` to Failure instances - - Applies function to the value if and only if this is a - :py:class:`Success`. - """ - constructor = type(self) - if self.failed(): - return constructor(function(self._value)) - else: - return self - - def get(self): - '''Gets the Success value if this is a Success otherwise throws an exception''' - if self.succeeded(): - return self._value - else: - raise NoSuchElementError('You cannot call `get` on a Failure, use `get_failure` instead') - - def get_failure(self): - '''Gets the Failure value if this is a Failure otherwise throws an exception''' - if self.failed(): - return self._value - else: - raise NoSuchElementError('You cannot call `get_failure` on a Success, use `get` instead') - - def get_or_else(self, default): - '''Returns the value from this Success or the given default argument if this is a Failure.''' - if self.succeeded(): - return self._value - else: - return default - - def succeeded(self): - """Return a Boolean that indicates if the value is an instance of Success - - >>> Success(True).succeeded() - True - >>> Failure('fubar').succeeded() - False - """ - return bool(self) - - def failed(self): - """Return a Boolean that indicates if the value is an instance of Failure - - >>> Failure('shit is fucked up').failed() - True - >>> Success('it worked!').failed() - False - """ - return not(bool(self)) - - @property - def message(self): - ''' - Return the message for the Try. If the ``message`` argument was provided to the constructor - that value is returned. Otherwise the string representation of the contained value is returened - ''' - if self._message is not None: - return self._message - else: - return str(self._value) - - @property - def start(self): - ''' - Start time of the operation in seconds since the UNIX epoch if specified in - the constructor or with the ``update`` method, ``None`` otherwise - ''' - return self._start - - @property - def end(self): - ''' - End time of the operation in seconds since the UNIX epoch if specified in - the constructor or with the ``update`` method, ``None`` otherwise - ''' - return self._end - - @property - def elapsed(self): - ''' - End time of the operation in seconds since the UNIX epoch if the start and end arguments - were specified in the constructor or with the ``update`` method, ``None`` otherwise - ''' - if self._end is None and self._start is None: - return None - - return self.end - self.start - - @property - def count(self): - '''Number of times the operation has been tried''' - return self._count - - def update(self, message=None, start=None, end=None, count=1): - ''' - Update the Try with new properties but the same value. Returns a new :class:`Failure` - or :py:class:`Success` and does not actually update in place. - - :param message: (optional) message to output to console if to_console is called. - If None, a string representation of the contained value is output. - Defaults to ``None`` - :param start: (optional) start time for the operation, in seconds since the UNIX epoch - `time.time()` is typically used for this value. Defaults to ``None``. - :type start: int - :param end: (optional) end time for the operation, in seconds since the UNIX epoch - ``time.time()`` is typically used for this value. Defaults to ``None``. - :type end: int - :param count: (optional) number of times the operation has been executed. Defaults to ``1`` - :type end: int - ''' - if (start is None and end is not None) or (end is None and start is not None): - raise InvalidTryError( - "The start and end argument must either be both None or not None") - - message = message or self._message - - # start = start or self._start does not work because start may == 0 and is therefore falsey - if start is None: - start = self._start - if end is None: - end = self._end - if count is None: - count = self._count - - constructor = type(self) - return constructor(self._value, message=message, start=start, end=end, count=count) - - def to_console(self, nl=True, exit_err=False, exit_status=1): - ''' - Write a message to the console. By convention, Success messages are written to stdout - and Failure messages are written to stderr. The Failure's `cause` is written to stderr - while the string repesentation of the Success's _value is written. - - :param message: the message to print - :param err: (optional) if set to true the file defaults to ``stderr`` instead of ``stdout``. - :param nl: (optional) if set to `True` (the default) a newline is printed afterwards. - :param exit_err: (optional) if set to True, exit the running program with a non-zero exit code - :param exit_status: (optional) the numeric exist status to return if exit is True - ''' - if self.succeeded(): - to_console(self.message, nl=nl) - else: - to_console(self.message, nl=nl, err=True, exit_err=exit_err, exit_status=exit_status) - - def fail_for_error(self, exit_status=1): - ''' - If a Failure, write the message to stderr and exit with return code of `exit_status` - Does nothing if a Success - - :param exit_status: (optional) the numeric exist status to return if exit is True - :type exit_status: int - ''' - if self.failed(): - to_console(self.message, nl=True, err=True, exit_err=True, - exit_status=exit_status) - - def raise_for_error(self, exception=FailureError): - ''' - Raise an exception if self is an instance of Failure. If the wrapped value is an - instance of Exeception or one of its subclasses, it is raised directly. The the - optional argument ``exception`` is specified, that type is raised with the wrapped - value as its argument. Otherwise, FailureError is raised. This method has no effect - is self is an instance of Success. - - :param exception: (optional) type of Exception to raise - ''' - - if self.succeeded(): - return - - wrapped_value = self.get_failure() - if isinstance(wrapped_value, Exception): - raise wrapped_value - else: - raise exception(wrapped_value) - - def filter(self, predicate): - ''' - If a Success, convert this to a Failure if the predicate is not satisfied. - Applies predicate to the wrapped value - - :param predicate: a function that takes the wrapped value as its argument and returns a boolean value - :rtype: :class:`Try ` object - :return: Try - ''' - if self.failed(): - return self - else: - wrapped_value = self.get() - if predicate(wrapped_value): - return self - else: - return Failure(wrapped_value) - - def __lt__(self, monad): - """Override to handle special case: Success.""" - if not isinstance(monad, (Failure, Success)): - fmt = "unorderable types: {} and {}'".format - raise TypeError(fmt(type(self), type(monad))) - if type(self) is type(monad): - # same type, either both lefts or rights, compare against value - return self._value < monad._value - if monad: - # self is Failure and monad is Success, left is less than right - return True - else: - return False - - def __repr__(self): - """Customize Show.""" - fmt = 'Success({})' if self else 'Failure({})' - return fmt.format(repr(self._value)) - - -class Failure(Try): - """Failure of :py:class:`Try`.""" - def __bool__(self): - # pylint: disable = no-self-use - return False - __nonzero__ = __bool__ - - -class Success(Try): - """Success of :py:class:`Try`.""" - def __bool__(self): - # pylint: disable = no-self-use - return True - - -class Maybe(Monad, Ord): - """A wrapper for values that be None - - >>> Some(42) - Some(42) - >>> Some([1, 2, 3]) - Some([1, 2, 3]) - >>> Some(Nothing) - Some(Nothing) - >>> Some(Some(2)) - Some(Some(2)) - >>> isinstance(Some(1), Maybe) - True - >>> isinstance(Nothing, Maybe) - True - >>> saving = 100 - >>> spend = lambda cost: Nothing if cost > saving else Some(saving - cost) - >>> spend(90) - Some(10) - >>> spend(120) - Nothing - >>> safe_div = lambda a, b: Nothing if b == 0 else Some(a / b) - >>> safe_div(12.0, 6) - Some(2.0) - >>> safe_div(12.0, 0) - Nothing - - Map operation with ``map``. Not that map only applies a function if the object is an - instance of Some. In the case of a Some, ``map`` returns the transformed value inside a Some. - No action is taken for a Nothing. - - >>> inc = lambda n: n + 1 - >>> Some(0) - Some(0) - >>> Some(0).map(inc) - Some(1) - >>> Some(0).map(inc).map(inc) - Some(2) - >>> Nothing.map(inc) - Nothing - - Comparison with ``==``, as long as what's wrapped inside are comparable. - - >>> Some(42) == Some(42) - True - >>> Some(42) == Nothing - False - >>> Nothing == Nothing - True - """ - @classmethod - def from_value(cls, value): - """Wraps ``value`` in a :class:`Maybe` monad. - - Returns a :class:`Some` if the value is evaluated as true. - :data:`Nothing` otherwise. - """ - return cls.unit(value) if value else Nothing - - def get(self): - '''Return the wrapped value if this is Some otherwise throws an exception''' - if self.is_empty(): - raise NoSuchElementError('You cannot call `get` on Nothing') - else: - return self._value - - def get_or_else(self, default): - '''Returns the value from this Some or the given default argument otherwise.''' - if self.is_empty(): - return default - else: - return self._value - - def filter(self, predicate): - ''' - Returns Some(value) if this is a Some and the value satisfies the given predicate. - - :param predicate: a function that takes the wrapped value as its argument and returns a boolean value - :rtype: :class:`Maybe ` object - :return: Maybe - ''' - if self.is_empty(): - return self - else: - wrapped_value = self.get() - if predicate(wrapped_value): - return self - else: - return Nothing - - def map(self, function): - """The map operation of :class:`Maybe`. - - Applies function to the value if and only if this is a :class:`Some`. - """ - constructor = type(self) - return self and constructor(function(self._value)) - - def is_empty(self): - '''Returns true, if this is None, otherwise false, if this is Some.''' - return self is Nothing - - def is_defined(self): - '''Returns true, if this is Some, otherwise false, if this is Nothing.''' - return self is not Nothing - - def __bool__(self): - return self is not Nothing - - __nonzero__ = __bool__ - - def __repr__(self): - """Customized Show.""" - if self is Nothing: - return 'Nothing' - else: - return 'Some({})'.format(repr(self._value)) - - def __iter__(self): - if self is not Nothing: - yield self._value - - -# pylint: disable = invalid-name -Some = Maybe -#: The :class:`Maybe` that represents nothing, a singleton, like ``None``. -Nothing = Maybe(Null) -Maybe.zero = Nothing -# pylint: enable = invalid-name - - -def _get_stacktrace(): - import traceback - if PY3: - from io import StringIO - else: - from StringIO import StringIO - - t, _, tb = sys.exc_info() - f = StringIO() - traceback.print_tb(tb, None, f) - stacktrace = f.getvalue() - return stacktrace - - -def try_out(callable, exception=None): - ''' - Executes a callable and wraps a raised exception in a Failure class. If an exception was - not raised, a Success is returned. If the keyword argument ``exception`` is not None, - only wrap the specified exception. Raise all other exceptions. - The stacktrace related to the exception is added to the wrapped exception - as the `stracktrace` property - - - :param callable: A callable reference, should return a value other than None - :rtype Try: a Success or Failure - ''' - - if exception is None: - catch_exception = Exception - else: - catch_exception = exception - - try: - return Success(callable()) - except catch_exception as e: - stacktrace = _get_stacktrace() - e.stacktrace = stacktrace - return Failure(e) - - -def to_console(message=None, nl=True, err=False, exit_err=False, exit_status=1): - ''' - Write a message to the console - - :param message: the message to print - :param err: (optional) if set to true the file defaults to ``stderr`` instead of ``stdout``. - :param nl: (optional) if set to `True` (the default) a newline is printed afterwards. - :param exit_err: (optional) if set to True, exit the running program with a non-zero exit code - :param exit_status: (optional) the numeric exist status to return if exit is True - ''' - if err: - stream = sys.stderr - else: - stream = sys.stdout - - stream.write(message) - - if nl: - stream.write(os.linesep) - - stream.flush() - if exit_err: - sys.exit(exit_status) - - -class SystemClock: - ''' - This is just a wrapper around the built-in time.time that makes it much easier to test - this module by mocking out time itself. - ''' - def __init__(self): - pass - - def time(self): - '''Returns value of current UNIX epoch in seconds''' - return time.time() - - def sleep(self, seconds): - time.sleep(seconds) - - -class StoppedClock: - ''' - This class only exists to make it easier to test retries - ''' - def __init__(self): - self.times = [] - - def set_times(self, times): - '''list of times for the self.time call to return - the times can be a single value or be a value + side effect to trigger - - example: - clock.set_times([100, 200, (300, lambda: Failure("Uh oh!"), 400]) - - the 3rd invocation of clock.time() will return the Failure - - example: - clock.set_times([100, 200, (300, function_with_side_effect), 400]) - - function_with_side_effect will be triggered the 3rd time clock.time() is invoked - ''' - self.times = times - self.current = iter(times) - - def sleep(self, seconds): - '''This sleep doesn't actually sleep, so your tests run quickly!''' - pass - - def time(self): - current_time = next(self.current) - if not isinstance(current_time, tuple): - return current_time - - current_time_val, side_effect = current_time - if isinstance(side_effect, Exception): - raise side_effect - - side_effect() - - return current_time_val - - -class Counter: - '''A simple counter''' - - def __init__(self, initial=0): - self._count = initial - - def increment(self): - self._count = self._count + 1 - - def reset(self): - self._count = 0 - - @property - def count(self): - return self._count - - -def tick_counter(column_limit=80): - - counter = Counter() - - def write_tick(log): - sys.stdout.write('.') - counter.increment() - # if we have reached the max # of columns, write a newline and reset the counter - if counter.count == column_limit: - sys.stdout.write(os.linesep) - counter.reset() - sys.stdout.flush() - - return write_tick - - -_clock = SystemClock() - - -class Again(Failure): - """ - Again of :py:class:`Failure`. - A handy alias of :py:class:`Failure` to indicate that an operation should be retried - """ - pass - - -class Stop(Success): - """ - Stop of :py:class:`Success`. - A handy alias of :py:class:`Success` to indicate that an operation should **not** be retried - """ - pass - - -class InvalidCallableError(Exception): - pass - - -def raise_if_invalid_result(result): - '''Raise InvalidCallableError if the result is not of type Try''' - if not isinstance(result, Try): - raise InvalidCallableError( - "Functions passed as arguments to the retry function must " - "return either tryme.Success, tryme.Failure, or raise an exception") - - -def retry_wrapper(acallable, timeout=300, delay=5, status_callback=None): - - @wraps(acallable) - def _retry(*args, **kwargs): - start = _clock.time() - assert timeout > 0, 'the timeout keyword argument must be greater than 0' - deadline = start + timeout - counter = Counter(0) - current_time = start - - while current_time < deadline: - counter.increment() - result = acallable(*args, **kwargs) - current_time = _clock.time() - end = current_time - raise_if_invalid_result(result) - - # update with time accounting - result = result.update(start=start, end=end, count=counter.count) - if result.succeeded(): - if status_callback: - status_callback(result) - return result - else: - if status_callback: - status_callback(result) - _clock.sleep(delay) - - return result.update(start=start, end=end, count=counter.count) - - return _retry - - -def retry(*args, **kwargs): - ''' - Function that wraps a callable with a retry loop. The callable should only return - :class:Failure, :class:Success, or raise an exception. This function can - be used as a decorator or directly wrap a function. This method returns a - a result object which is an instance of py:class:`Success` or py:class:`Failure`. - This function updates the result with the time of the first attempt, the time - of the last attempt, and the total count of attempts - - :param acallable: object that can be called - :type acallable: function - :param timeout: (optional) maximum period, in seconds, to wait until an individual try succeeds. - Defaults to ``300`` seconds - :type timeout: int - :param delay: (optional) delay between retries in seconds. Defaults to ``5`` seconds. - :type delay: int - :param status_callback: (optional) callback to invoke after each retry, is passed the result - as an argument. Defaults to ``None``. - :type status_callback: function - - Usage:: - >>> deadline = time.time() + 300 - >>> dinner_iterator = iter([False, False, True]) - >>> def dinner_is_ready(): - ... return next(dinner_iterator) - >>> breakfast_iterator = iter([False, False, True]) - >>> def breakfast_is_ready(): - ... return next(breakfast_iterator) - >>> @retry - ... def wait_for_dinner(): - ... if dinner_is_ready(): - ... return Success("Ready!") - ... else: - ... return Failure("not ready yet") - >>> result = wait_for_dinner() # doctest: +SKIP - >>> result # doctest: +SKIP - Success("Ready!") - >>> result.elapsed # doctest: +SKIP - 8 - >>> result.count # doctest: +SKIP - 3 - >>> @retry - ... def wait_for_breakfast(): - ... if breakfast_is_ready(): - ... return Success("Ready!") - ... else: - ... return Failure("not ready yet") - >>> result = wait_for_breakfast() # doctest: +SKIP - Success("Ready!") - >>> result.elapsed # doctest: +SKIP - 8 - >>> result.count # doctest: +SKIP - 3 - - The names py:class:`Success` and py:class:`Failure` do not always - map well to operations that need to be retried. The subclasses - py:class:`Stop` and py:class:`Again` can be more intuitive.:: - >>> breakfast_iterator = iter([False, False, True]) - >>> def breakfast_is_ready(): - ... return next(breakfast_iterator) - >>> @retry - ... def wait_for_breakfast(): - ... if breakfast_is_ready(): - ... return Stop("Ready!") - ... else: - ... return Again("not ready yet") - - ''' - - # if used as a decorator without arguments `@retry`, the first argument is - # is the decorated function - # If used as a decorator with keyword arguments, say `@retry(timeout=900)` - # the args are empty and the decorated function is supplied sometime later - # as the argument to the decorator. Confusing! - acallable = None - if len(args) > 0: - acallable = args[0] - - if acallable is not None: - return retry_wrapper(acallable, **kwargs) - else: - def decorator(func): - return retry_wrapper(func, **kwargs) - - return decorator From 247c2f7649fd1fb51a3f35ca9a0ce68fed2cbdfb Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Mon, 14 Sep 2026 20:21:55 +0000 Subject: [PATCH 13/18] Remove test comments --- specs/default/cluster-init/activate-hook.sh | 1 - specs/default/cluster-init/configure-hook.sh | 1 - specs/default/cluster-init/roles/server-install.sh | 1 - 3 files changed, 3 deletions(-) diff --git a/specs/default/cluster-init/activate-hook.sh b/specs/default/cluster-init/activate-hook.sh index 6dfd1c6..7038e62 100644 --- a/specs/default/cluster-init/activate-hook.sh +++ b/specs/default/cluster-init/activate-hook.sh @@ -1,6 +1,5 @@ #!/bin/bash source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -echo "im running activate hook" ROLE=$(jetpack config pbspro.role "") || fail case "$ROLE" in diff --git a/specs/default/cluster-init/configure-hook.sh b/specs/default/cluster-init/configure-hook.sh index 0bf135e..b45c330 100644 --- a/specs/default/cluster-init/configure-hook.sh +++ b/specs/default/cluster-init/configure-hook.sh @@ -1,6 +1,5 @@ #!/bin/bash source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -echo "im running configure hook" ROLE=$(jetpack config pbspro.role "") || fail diff --git a/specs/default/cluster-init/roles/server-install.sh b/specs/default/cluster-init/roles/server-install.sh index 5846ac6..12a02ab 100644 --- a/specs/default/cluster-init/roles/server-install.sh +++ b/specs/default/cluster-init/roles/server-install.sh @@ -2,7 +2,6 @@ source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail -echo "hi im the server node" CLUSTER_NAME=$(jq -r .cluster "$CONFIG_PATH") || fail CONNECTION_URL=$(jq -r .url "$CONFIG_PATH") || fail IGNORE_WORKQ=$(jetpack config pbspro.queues.workq.ignore "False") || fail From a832af5f831e65a17c8b0e3f53258fd19fc6ab04 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Tue, 15 Sep 2026 19:56:54 +0000 Subject: [PATCH 14/18] Update cyclecloud api wheel in release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e37a2f2..69f42c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,7 @@ jobs: generate_release_notes: true files: | blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.VERSION }}.tar.gz - blobs/cyclecloud_api-8.3.1-py2.py3-none-any.whl + blobs/cyclecloud_api-8.10.0-py2.py3-none-any.whl blobs/hwloc-libs-1.11.9-3.el8.x86_64.rpm blobs/openpbs-client-20.0.1-0.x86_64.rpm blobs/openpbs-client-22.05.11-0.x86_64.rpm From d1b94971bf400c872a0dbeeb6dbadda56ff96a12 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Tue, 15 Sep 2026 20:24:22 +0000 Subject: [PATCH 15/18] Add back azpbs.env usage --- specs/default/cluster-init/files/utils.sh | 31 +++++++++++++++++++ .../cluster-init/roles/execute-install.sh | 9 +++--- .../cluster-init/roles/login-install.sh | 7 ++--- .../cluster-init/roles/server-install.sh | 10 ++++++ 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/specs/default/cluster-init/files/utils.sh b/specs/default/cluster-init/files/utils.sh index 9a0f574..492d45b 100644 --- a/specs/default/cluster-init/files/utils.sh +++ b/specs/default/cluster-init/files/utils.sh @@ -36,3 +36,34 @@ function get_package_name() { echo "$package_name" fi } + +function get_server_hostname() { + local server_hostname=$(jetpack config pbspro.scheduler "") || fail + local cluster_name=$(jq -r .cluster "$CONFIG_PATH") || fail + + if [[ -z "$server_hostname" ]]; then + local -r max_retries=10 + local -r retry_delay=15 + local attempt=1 + + # Read server hostname from azpbs.env config file generated by server node + if [[ ! -e "/sched/${cluster_name}/azpbs.env" ]]; then + while [[ $attempt -lt $max_retries ]]; do + sleep $retry_delay + ((attempt+=1)) + + if [[ -e "/sched/${cluster_name}/azpbs.env" ]]; then + break; + fi + done + + if [[ $attempt == $max_retries ]]; then + fail "Failed to read /sched/${cluster_name}/azpbs.env after $max_retries attempts. Exiting." + fi + fi + source "/sched/${cluster_name}/azpbs.env" || fail + echo "$PBS_SCHEDULER_HOSTNAME" + else + echo "$server_hostname" + fi +} diff --git a/specs/default/cluster-init/roles/execute-install.sh b/specs/default/cluster-init/roles/execute-install.sh index 02b94a8..3a086b2 100755 --- a/specs/default/cluster-init/roles/execute-install.sh +++ b/specs/default/cluster-init/roles/execute-install.sh @@ -3,20 +3,21 @@ source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 EXECUTE_HOSTNAME=$(jetpack config hostname) || fail -SERVER_IP_ADDRESS=$(jetpack config cyclecloud.mounts.nfs_sched.address "") || fail +SERVER_HOSTNAME=$(get_server_hostname) || fail + # Forces execute node's hostname to be updated (scalelib is blocked until the hostname is correct) # TODO: this installation status should be done by jetpack before cluster-inits are run "${CYCLECLOUD_HOME}/system/embedded/bin/python" -c "import jetpack.converge as jc; jc._send_installation_status('warning')" -if [[ -n "$SERVER_IP_ADDRESS" ]]; then - echo "$SERVER_IP_ADDRESS" > /var/spool/pbs/server_name +if [[ -n "$SERVER_HOSTNAME" ]]; then + echo "$SERVER_HOSTNAME" > /var/spool/pbs/server_name chmod 0644 /var/spool/pbs/server_name || fail cp "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/mom_config.template" /var/spool/pbs/mom_priv/config || fail chmod 0644 /var/spool/pbs/mom_priv/config || fail - sed -e "s|__SERVERNAME__|${SERVER_IP_ADDRESS}|g" \ + sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail chmod 0644 /etc/pbs.conf || fail fi diff --git a/specs/default/cluster-init/roles/login-install.sh b/specs/default/cluster-init/roles/login-install.sh index 8f31146..f156fed 100755 --- a/specs/default/cluster-init/roles/login-install.sh +++ b/specs/default/cluster-init/roles/login-install.sh @@ -2,11 +2,10 @@ source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -SERVER_IP_ADDRESS=$(jetpack config cyclecloud.mounts.nfs_sched.address "") || fail +SERVER_HOSTNAME=$(get_server_hostname) || fail - -if [[ -n "$SERVER_IP_ADDRESS" ]]; then - sed -e "s|__SERVERNAME__|${SERVER_IP_ADDRESS}|g" \ +if [[ -n "$SERVER_HOSTNAME" ]]; then + sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail chmod 0644 /etc/pbs.conf || fail fi diff --git a/specs/default/cluster-init/roles/server-install.sh b/specs/default/cluster-init/roles/server-install.sh index 12a02ab..4b03ec0 100644 --- a/specs/default/cluster-init/roles/server-install.sh +++ b/specs/default/cluster-init/roles/server-install.sh @@ -9,7 +9,17 @@ IGNORE_HTCQ=$(jetpack config pbspro.queues.htcq.ignore "False") || fail CRON_METHOD=$(jetpack config pbspro.cron_method "pbs_cron") || fail PBSPRO_AUTOSCALE_PROJECT_HOME="/opt/cycle/pbspro" PBSPRO_AUTOSCALE_INSTALLER="cyclecloud-pbspro-pkg-${PBSPRO_AUTOSCALE_VERSION}.tar.gz" +PACKAGE_NAME=$(get_package_name "server") || fail +mkdir -p "/sched/${CLUSTER_NAME}" || fail + +cat << EOF > "/sched/${CLUSTER_NAME}/azpbs.env" +#!/bin/bash +PBS_SCHEDULER_HOSTNAME=$(hostname) +PBS_SCHEDULER_IP=$(hostname -i) + +EOF +chmod a+r "/sched/${CLUSTER_NAME}/azpbs.env" || fail mkdir -p -m 0755 "$PBSPRO_AUTOSCALE_PROJECT_HOME" || fail From 22a401781ada968565dc9848531af47e00a3db60 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Tue, 15 Sep 2026 20:49:29 +0000 Subject: [PATCH 16/18] Update cc api url for blob download --- package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.py b/package.py index c7e8456..f172d76 100644 --- a/package.py +++ b/package.py @@ -39,7 +39,7 @@ def get_cycle_packages(args: Namespace) -> List[str]: scalelib_url = f"https://github.com/Azure/cyclecloud-scalelib/archive/refs/tags/{SCALELIB_VERSION}.tar.gz" - cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/{cyclecloud_api_file}" #TODO: ensure this is the correct url + cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2.0.26/{cyclecloud_api_file}" to_download = { scalelib_file: (args.scalelib, scalelib_url), cyclecloud_api_file: (args.cyclecloud_api, cyclecloud_api_url), From 11850eaf55269fdae9f15b6a4f6cef84fbd44bbe Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Thu, 17 Sep 2026 15:11:38 +0000 Subject: [PATCH 17/18] Bump scalelib version to 1.0.12 --- package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.py b/package.py index f172d76..593a622 100644 --- a/package.py +++ b/package.py @@ -11,7 +11,7 @@ from typing import Dict, List, Optional from util import download_release_files -SCALELIB_VERSION = "1.0.11" +SCALELIB_VERSION = "1.0.12" CYCLECLOUD_API_VERSION = "8.10.0" From c4ee11ddcb9d1edbdb6214a5c8ca9ff224219f17 Mon Sep 17 00:00:00 2001 From: Jelena Gvero Date: Thu, 17 Sep 2026 16:12:14 +0000 Subject: [PATCH 18/18] Fix bug with external /sched filesystem and support lustre in template --- templates/openpbs.txt | 226 ++++++++++++++++++++++++++++++------------ 1 file changed, 165 insertions(+), 61 deletions(-) diff --git a/templates/openpbs.txt b/templates/openpbs.txt index 5f2394d..70f33e3 100644 --- a/templates/openpbs.txt +++ b/templates/openpbs.txt @@ -17,6 +17,10 @@ Autoscale = $Autoscale Region = $Region KeyPairLocation = ~/.ssh/cyclecloud.pem Azure.Identities = $ManagedIdentity + + # Lustre mounts require termination notifications to unmount + EnableTerminateNotification = ${NFSType == "lustre" || NFSSchedType == "lustre" || AdditionalNFSType == "lustre" || EnableTerminateNotification} + TerminateNotificationTimeout = 10m [[[configuration]]] pbspro.version = $PBSVersion @@ -47,25 +51,27 @@ Autoscale = $Autoscale SSD = True [[[configuration cyclecloud.mounts.nfs_shared]]] - type = nfs + type = $NFSType mountpoint = /shared - export_path = $NFSSharedExportPath + export_path = ${ifThenElse(NFSType == "lustre", strcat("tcp:/lustrefs", NFSSharedExportPath), NFSSharedExportPath)} address = $NFSAddress options = $NFSSharedMountOptions node_name = server [[[configuration cyclecloud.mounts.nfs_sched]]] - type = nfs + type = $NFSSchedType mountpoint = /sched - disabled = $NFSSchedDisable + export_path = ${ifThenElse(NFSSchedType == "lustre", strcat("tcp:/lustrefs", NFSSchedExportPath), NFSSchedExportPath)} + address = $NFSSchedAddress #note + options = $NFSSchedMountOptions node_name = server [[[configuration cyclecloud.mounts.additional_nfs]]] - disabled = ${AdditionalNAS isnt true} - type = nfs - address = $AdditonalNFSAddress + disabled = ${AdditionalNFS isnt true} + type = $AdditionalNFSType + address = $AdditionalNFSAddress mountpoint = $AdditionalNFSMountPoint - export_path = $AdditionalNFSExportPath + export_path = ${ifThenElse(AdditionalNFSType == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath), AdditionalNFSExportPath)} options = $AdditionalNFSMountOptions @@ -77,8 +83,8 @@ Autoscale = $Autoscale [[[configuration]]] cyclecloud.discoverable=true - cyclecloud.mounts.nfs_sched.disabled = true - cyclecloud.mounts.nfs_shared.disabled = ${NFSType != "External"} + cyclecloud.mounts.nfs_sched.disabled = $UseBuiltinSched + cyclecloud.mounts.nfs_shared.disabled = $UseBuiltinShared pbspro.cron_method = $AzpbsCronMethod pbspro.queues.workq.ignore = ${Autoscale != true} pbspro.queues.htcq.ignore = ${Autoscale != true} @@ -90,34 +96,38 @@ Autoscale = $Autoscale AssociatePublicIpAddress = $UsePublicNetwork [[[volume sched]]] - Size = 1024 + Size = $SchedFilesystemSize SSD = True Mount = builtinsched - Persistent = False + Persistent = True #TODO: does this need to be true + Disabled = ${!UseBuiltinSched} [[[volume shared]]] - Size = ${ifThenElse(NFSType == "Builtin", FilesystemSize, 2)} + Size = $FilesystemSize SSD = True Mount = builtinshared - Persistent = ${NFSType == "Builtin"} + Persistent = True #TODO: does this need to be true + Disabled = ${!UseBuiltinShared} [[[configuration cyclecloud.mounts.builtinsched]]] + disabled = ${!UseBuiltinSched} mountpoint = /sched fs_type = xfs [[[configuration cyclecloud.mounts.builtinshared]]] - disabled = ${NFSType != "Builtin"} + disabled = ${!UseBuiltinShared} mountpoint = /shared fs_type = xfs [[[configuration cyclecloud.exports.builtinsched]]] + disabled = ${!UseBuiltinSched} export_path = /sched options = no_root_squash samba.enabled = false type = nfs [[[configuration cyclecloud.exports.builtinshared]]] - disabled = ${NFSType != "Builtin"} + disabled = ${!UseBuiltinShared} export_path = /shared samba.enabled = false type = nfs @@ -237,61 +247,139 @@ Order = 10 [parameters Network Attached Storage] Order = 15 + [[parameters Shared Storage]] + Order = 10 + + [[[parameter About Shared Storage]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

The directories /sched and /shared are network attached mounts and exist on all nodes of the cluster.
+
+ Options for providing these mounts:
+ [Builtin]: The server node is an NFS server that provides the mountpoint to the other nodes of the cluster.
+ [External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server provides the mountpoint.
+ [Azure Managed Lustre]: An Azure Managed Lustre deployment provides the mountpoint.
+

+

Note: the cluster must be terminated for changes to filesystem mounts to take effect.

''' + Conditions.Hidden := false [[parameters Scheduler Mount]] - Order = 5 + Order = 20 + Label = File-system Mount for /sched + [[[parameter About sched]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template = '''

The directory /sched is a network attached mount and exists in all nodes of the cluster. - It's managed by the scheduler node. - To disable the mount of the /sched directory, and to supply your own for a hybrid scenario, select the checkbox below.''' + Config.Template = '''

OpenPBS configuration shared with cluster nodes is stored in the /sched directory. It is managed by the server node.

''' Order = 6 - [[[parameter NFSSchedDisable]]] + [[[parameter About sched part 2]]] HideLabel = true - DefaultValue = false - Widget.Plugin = pico.form.BooleanCheckBox - Widget.Label = External Scheduler + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Uncheck the box below to disable the built-in NFS export of the /sched directory and use an external file-system.

''' + Order = 7 - [[parameters Default NFS Share]] - Order = 10 - [[[parameter About shared]]] + [[[parameter UseBuiltinSched]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /sched + DefaultValue = true + ParameterType = Boolean + + [[[parameter NFSSchedDiskWarning]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

The directory /shared is a network attached mount and exists in all nodes of the cluster. Users' home directories reside within this mountpoint with the base homedir /shared/home.

There are two options for providing this mount:
[Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster.
[External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server, provides the mountpoint.

Note: the cluster must be terminated for this to take effect.

" - Order = 20 + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinSched - [[[parameter NFSType]]] - Label = NFS Type + [[[parameter NFSSchedType]]] + Label = FS Type ParameterType = StringList - Config.Label = Type of NFS to use for this cluster + Config.Label = Type of shared filesystem to use for this cluster Config.Plugin = pico.form.Dropdown - Config.Entries := {[Label="External NFS"; Value="External"], [Label="Builtin"; Value="Builtin"]} - DefaultValue = Builtin + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedAddress]]] + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedExportPath]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /sched + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedMountOptions]]] + Label = Mount Options + Description = File System Client Mount Options + Conditions.Hidden := UseBuiltinSched + + [[[parameter SchedFilesystemSize]]] + Label = Size (GB) + Description = The filesystem size (cannot be changed after initial start) + DefaultValue = 1024 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := !UseBuiltinSched + + [[parameters Default NFS Share]] + Order = 30 + Label = File-system Mount for /shared + + [[[parameter About shared]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Users' home directories reside within the /shared mountpoint with the base homedir /shared/home.

''' + Order = 6 + + [[[parameter About shared part 2]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Uncheck the box below to disable the built-in NFS export of the /shared directory and use an external file-system.

''' + Order = 7 + + [[[parameter UseBuiltinShared]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /shared + DefaultValue = true + ParameterType = Boolean [[[parameter NFSDiskWarning]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Warning: switching an active cluster over to NFS will delete the shared disk.

" - Conditions.Hidden := NFSType != "External" + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinShared + + [[[parameter NFSType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Type of shared filesystem to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinShared [[[parameter NFSAddress]]] - Label = NFS IP Address - Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String - Conditions.Hidden := NFSType != "External" + Conditions.Hidden := UseBuiltinShared [[[parameter NFSSharedExportPath]]] - Label = Shared Export Path + Label = Export Path Description = The path exported by the file system DefaultValue = /shared - Conditions.Hidden := NFSType != "External" + Conditions.Hidden := UseBuiltinShared [[[parameter NFSSharedMountOptions]]] - Label = NFS Mount Options - Description = NFS Client Mount Options - Conditions.Hidden := NFSType != "External" + Label = Mount Options + Description = File system client mount options + Conditions.Hidden := UseBuiltinShared [[[parameter FilesystemSize]]] Label = Size (GB) @@ -302,44 +390,56 @@ Order = 15 Config.MinValue = 10 Config.MaxValue = 10240 Config.IntegerOnly = true - Conditions.Excluded := NFSType != "Builtin" + Conditions.Excluded := !UseBuiltinShared [[parameters Additional NFS Mount]] - Order = 20 - [[[parameter Additional NFS Mount Readme]]] + Order = 40 + Label = Additional Filesystem Mount + + [[[parameter Additional Shared FS Mount Readme]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Mount another NFS endpoint on the cluster nodes.

" + Config.Template := "

Mount another shared file-system endpoint on the cluster nodes.

" Order = 20 - [[[parameter AdditionalNAS]]] + [[[parameter AdditionalNFS]]] HideLabel = true DefaultValue = false Widget.Plugin = pico.form.BooleanCheckBox - Widget.Label = Add NFS mount + Widget.Label = Add Shared Filesystem mount - [[[parameter AdditonalNFSAddress]]] - Label = NFS IP Address - Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + [[[parameter AdditionalNFSType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Shared filesystem type of the additional mount + Config.Plugin = pico.form.Dropdown + + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Excluded := AdditionalNFS isnt true + + [[[parameter AdditionalNFSAddress]]] + Label = IP Address + Description = The IP address or hostname of the additional mount. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSMountPoint]]] - Label = NFS Mount Point + Label = Mount Point Description = The path at which to mount the Filesystem DefaultValue = /data - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSExportPath]]] - Label = NFS Export Path + Label = Export Path Description = The path exported by the file system DefaultValue = /data - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSMountOptions]]] - Label = NFS Mount Options - Description = NFS Client Mount Options - Conditions.Excluded := AdditionalNAS isnt true + Label = Mount Options + Description = Filesystem Client Mount Options + Conditions.Excluded := AdditionalNFS isnt true @@ -369,6 +469,10 @@ Order = 20 Config.Increment = 64 DefaultValue = 0 + [[[parameter EnableTerminateNotification]]] + Label = Enable Termination notifications + DefaultValue = False + [[parameters Software]]