diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac8d5a6..69f42c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,16 +10,19 @@ jobs: build: name: Upload Release Asset runs-on: ubuntu-latest + container: + image: almalinux:8 permissions: contents: write steps: + - name: Install build prerequisites + run: | + dnf update -y + dnf install -y python3.11 python3.11-pip git tar gzip which findutils - name: Checkout code uses: actions/checkout@v4 - name: Build pkg - run: | - sudo apt update || apt update - sudo apt-get install -y python3 python3-pip || apt-get install -y python3 python3-pip - ./build.sh + run: ./build.sh - name: Get the version id: get_version run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT @@ -41,7 +44,7 @@ jobs: generate_release_notes: true files: | blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.VERSION }}.tar.gz - blobs/cyclecloud_api-8.3.1-py2.py3-none-any.whl + blobs/cyclecloud_api-8.10.0-py2.py3-none-any.whl blobs/hwloc-libs-1.11.9-3.el8.x86_64.rpm blobs/openpbs-client-20.0.1-0.x86_64.rpm blobs/openpbs-client-22.05.11-0.x86_64.rpm diff --git a/.gitignore b/.gitignore index 0174ffc..5741ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ dist/** .env libs/** venv/** +.build.log diff --git a/README.md b/README.md index 1dac053..d8f6a17 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,6 @@ cd cyclecloud-pbspro # If you have jetpack available, you may use the following: # ./generate_autoscale_json.sh --install-dir /opt/cycle/pbspro \ -# --username $(jetpack config cyclecloud.config.username) \ -# --password $(jetpack config cyclecloud.config.password) \ # --url $(jetpack config cyclecloud.config.web_server) \ # --cluster-name $(jetpack config cyclecloud.cluster.name) diff --git a/build.sh b/build.sh index 6719837..536c4cc 100755 --- a/build.sh +++ b/build.sh @@ -1,15 +1,16 @@ #!/bin/bash +set -e # create a new venv if it does not exist, or is older than 7 days -if [ ! $(find . -path ./venv/created -mtime -7) ]; then +if [ -z "$(find . -path ./venv/created -mtime -7 -print -quit)" ] || + ! venv/bin/python -c 'import sys; raise SystemExit(sys.version_info < (3, 11))' 2>/dev/null; then rm -rf venv python3 -m venv venv source venv/bin/activate - pip install setuptools + python3 -m pip install setuptools touch venv/created else source venv/bin/activate fi -python package.py - +./docker-package.sh \ No newline at end of file diff --git a/docker-package.sh b/docker-package.sh new file mode 100755 index 0000000..15bca3a --- /dev/null +++ b/docker-package.sh @@ -0,0 +1,85 @@ +#!/bin/bash +echo see .build.log for more information +log_file=".build.log" +check_dirty_changes() { + if [ -n "$(git status --porcelain)" ]; then + echo "Error: There are uncommitted changes in the current branch. Please commit or stash them before running this script." + exit 1 + fi +} + +# Call the function to check for dirty changes +# check_dirty_changes + +print_branch_and_last_commit() { + branch=$(git rev-parse --abbrev-ref HEAD) + last_commit=$(git log -1 --pretty=format:"%h - %s (%ci)") + + printf "%-20s: %s\n" "Current branch" "$branch" + printf "%-20s: %s\n" "Last commit" "$last_commit" +} + +delete_existing_blobs() { + printf "%-20s: %s\n" "Deleting existing Blob Files" "" + while IFS= read -r file; do + if [ -f "blobs/$file" ]; then + printf "%-20s: %s\n" "" "$file" + rm -f "blobs/$file" || exit 1 + fi + done < <(awk -F' *= *' '/^\[blobs\]/ {found=1} found && /^Files/ {gsub(/, */, "\n", $2); print $2; exit}' project.ini) +} + +check_blobs_files_exist() { + local version="$1" + local missing_files=0 + + printf "%-20s: %s\n" "Blob Files" "" + while IFS= read -r file; do + printf "%-20s: %s\n" "" "$file" + if [ ! -f "blobs/$file" ]; then + echo "Error: File blobs/$file does not exist." + missing_files=1 + fi + done < <(awk -F' *= *' '/^\[blobs\]/ {found=1} found && /^Files/ {gsub(/, */, "\n", $2); print $2; exit}' project.ini) + + if [ $missing_files -eq 1 ]; then + echo "One or more required files are missing in the blobs directory." + exit 1 + + fi +} + +get_version_from_project_ini() { + version=$(awk -F' *= *' '/^\[project\]/ {found=1} found && /^version/ {print $2; exit}' project.ini) + printf "%-20s: %s\n" "Project Version" "$version" + check_blobs_files_exist "$version" +} + +local_azpbs=/source/ +if [ "$1" != "" ]; then + scalelib=$(realpath $1) + local_scalelib=/source/cyclecloud-scalelib + extra_args="-v ${scalelib}:${local_scalelib}" +fi + +if command -v docker; then + runtime=docker + runtime_args= +elif command -v podman; then + runtime=podman + runtime_args="--privileged" +else + echo "`docker` or `podman` binary not found. Install docker or podman to build RPMs with this script" + exit 1 +fi + +{ + delete_existing_blobs + # allows caching + $runtime build -t azpbs_build:latest -f util/Dockerfile . + $runtime run --user "$(id -u):$(id -g)" -e HOME=/tmp -v "$(pwd):${local_azpbs}" $runtime_args $extra_args -ti azpbs_build:latest /bin/bash ${local_azpbs}/util/build.sh $local_scalelib +} &> $log_file + +# Call the function to print the branch and the last commit +print_branch_and_last_commit +get_version_from_project_ini diff --git a/docker-rpmbuild.sh b/docker-rpmbuild.sh index 30d74ec..41c3fdc 100755 --- a/docker-rpmbuild.sh +++ b/docker-rpmbuild.sh @@ -1,7 +1,7 @@ #!/bin/bash if command -v docker; then - docker run -v $(pwd)/specs/default/cluster-init/files:/source -v $(pwd)/blobs:/root/rpmbuild/RPMS/x86_64 -ti centos:7 /bin/bash -e /source/00-build-pbspro.sh + docker run -v $(pwd)/specs/default/cluster-init/files:/source -v $(pwd)/blobs:/root/rpmbuild/RPMS/x86_64 -ti almalinux:8 /bin/bash -e /source/00-build-pbspro.sh else echo "`docker` binary not found. Install docker to build RPMs with this script" fi diff --git a/generate_autoscale_json.sh b/generate_autoscale_json.sh index 0a152ab..d06f0f3 100755 --- a/generate_autoscale_json.sh +++ b/generate_autoscale_json.sh @@ -28,7 +28,7 @@ CLUSTER_NAME= IGNORE_QUEUES_ARG= function usage() { - echo Usage: $0 --username username --password password --url https://fqdn:port --cluster-name cluster_name [--install-dir /opt/cycle/pbspro] + echo Usage: $0 --url https://fqdn:port --cluster-name cluster_name [--username username] [--password password] [--install-dir /opt/cycle/pbspro] exit 2 } @@ -73,7 +73,9 @@ done if [ "$1" == "-h" ]; then usage; fi if [ "$1" == "-help" ]; then usage; fi +if [ "$USERNAME" == "" ]; then USERNAME=$(jetpack config cyclecloud.config.username); fi if [ "$USERNAME" == "" ]; then usage; fi +if [ "$PASSWORD" == "" ]; then PASSWORD=$(jetpack config cyclecloud.config.password); fi if [ "$PASSWORD" == "" ]; then usage; fi if [ "$URL" == "" ]; then usage; fi if [ "$CLUSTER_NAME" == "" ]; then usage; fi diff --git a/install.sh b/install.sh index 58e53c3..22b2ae2 100755 --- a/install.sh +++ b/install.sh @@ -56,27 +56,37 @@ echo VENV=$VENV # remove jetpack's python3 from the path export PATH=$(echo "$PATH" | sed -e 's/\/opt\/cycle\/jetpack\/system\/embedded\/bin://g' | sed -e 's/:\/opt\/cycle\/jetpack\/system\/embedded\/bin//g') set +e -which python3 > /dev/null; +which python3.11 > /dev/null; if [ $? != 0 ]; then if [ $INSTALL_PYTHON3 == 1 ]; then - yum install -y -q python3 || exit 1 + yum install -y -q python3.11 || exit 1 else - echo Please install python3 >&2; + echo Please install python3.11 >&2; + exit 1 + fi +fi + +python3.11 -m pip --version > /dev/null 2>&1 +if [ $? != 0 ]; then + if [ $INSTALL_PYTHON3 == 1 ]; then + yum install -y -q python3.11-pip || exit 1 + else + echo Please install pip for python3.11 >&2; exit 1 fi fi set -e if [ $INSTALL_VIRTUALENV == 1 ]; then - python3 -m pip install -q virtualenv + python3.11 -m pip install -q virtualenv fi set +e -python3 -m virtualenv --version 2>&1 > /dev/null +python3.11 -m virtualenv --version 2>&1 > /dev/null if [ $? != 0 ]; then if [ $INSTALL_VIRTUALENV ]; then - python3 -m pip install -q virtualenv || exit 1 + python3.11 -m pip install -q virtualenv || exit 1 else echo Please install virtualenv for python3 >&2 exit 1 @@ -84,7 +94,7 @@ if [ $? != 0 ]; then fi set -e -python3 -m virtualenv $VENV +python3.11 -m virtualenv $VENV source "${VENV}/bin/activate" # not sure why but pip gets confused installing frozendict locally # if you don't install it first. It has no dependencies so this is safe. diff --git a/package.py b/package.py index d917b8f..593a622 100644 --- a/package.py +++ b/package.py @@ -11,8 +11,8 @@ from typing import Dict, List, Optional from util import download_release_files -SCALELIB_VERSION = "1.0.11" -CYCLECLOUD_API_VERSION = "8.3.1" +SCALELIB_VERSION = "1.0.12" +CYCLECLOUD_API_VERSION = "8.10.0" def build_sdist() -> str: @@ -39,7 +39,7 @@ def get_cycle_packages(args: Namespace) -> List[str]: scalelib_url = f"https://github.com/Azure/cyclecloud-scalelib/archive/refs/tags/{SCALELIB_VERSION}.tar.gz" - cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/{cyclecloud_api_file}" + cyclecloud_api_url = f"https://github.com/Azure/cyclecloud-pbspro/releases/download/2.0.26/{cyclecloud_api_file}" to_download = { scalelib_file: (args.scalelib, scalelib_url), cyclecloud_api_file: (args.cyclecloud_api, cyclecloud_api_url), @@ -119,7 +119,7 @@ def _add(name: str, path: Optional[str] = None, mode: Optional[int] = None) -> N _add("packages/" + dep, dep_path) packages.append(dep_path) - check_call(["pip", "download"] + packages, cwd=build_dir) + check_call([sys.executable, "-m", "pip", "download"] + packages, cwd=build_dir) print("Using build dir", build_dir) by_package: Dict[str, List[str]] = {} diff --git a/package.sh b/package.sh new file mode 100755 index 0000000..20eedf1 --- /dev/null +++ b/package.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e +cd $(dirname $0)/ + + +if [ ! -e libs ]; then + mkdir libs +fi + +LOCAL_SCALELIB=$1 + +rm -f dist/* + +if [ "$LOCAL_SCALELIB" == "" ]; then + # we are using released versions of scalelib + python3.11 package.py +else + pushd $LOCAL_SCALELIB + rm -f dist/*.gz + # python3 setup.py swagger + python3.11 setup.py sdist + popd + # swagger=`ls $LOCAL_SCALELIB/dist/swagger*.gz` + scalelib=`ls $LOCAL_SCALELIB/dist/cyclecloud_scalelib*.gz` + # python3 package.py --scalelib $scalelib --swagger $swagger + python3.11 package.py --scalelib $scalelib +fi diff --git a/project.ini b/project.ini index 35e0a24..3c3539a 100644 --- a/project.ini +++ b/project.ini @@ -6,7 +6,7 @@ version = 2.0.26 autoupgrade = true [blobs] -Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, cyclecloud_api-8.3.1-py2.py3-none-any.whl, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm +Files = cyclecloud-pbspro-pkg-2.0.26.tar.gz, cyclecloud_api-8.10.0-py2.py3-none-any.whl, hwloc-libs-1.11.9-3.el8.x86_64.rpm, openpbs-client-20.0.1-0.x86_64.rpm, openpbs-client-22.05.11-0.x86_64.rpm, openpbs-execution-20.0.1-0.x86_64.rpm, openpbs-execution-22.05.11-0.x86_64.rpm, openpbs-server-20.0.1-0.x86_64.rpm, openpbs-server-22.05.11-0.x86_64.rpm, pbspro-client-18.1.4-0.x86_64.rpm, pbspro-debuginfo-18.1.4-0.x86_64.rpm, pbspro-execution-18.1.4-0.x86_64.rpm, pbspro-server-18.1.4-0.x86_64.rpm [spec server] diff --git a/specs/default/cluster-init/activate-hook.sh b/specs/default/cluster-init/activate-hook.sh new file mode 100644 index 0000000..7038e62 --- /dev/null +++ b/specs/default/cluster-init/activate-hook.sh @@ -0,0 +1,9 @@ +#!/bin/bash +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 +ROLE=$(jetpack config pbspro.role "") || fail + +case "$ROLE" in + execute) systemctl start pbs || exit 1 ;; + server|login) ;; + *) fail "Unknown pbspro.role '$ROLE'" ;; +esac diff --git a/specs/default/cluster-init/configure-hook.sh b/specs/default/cluster-init/configure-hook.sh new file mode 100644 index 0000000..b45c330 --- /dev/null +++ b/specs/default/cluster-init/configure-hook.sh @@ -0,0 +1,10 @@ +#!/bin/bash +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 + +ROLE=$(jetpack config pbspro.role "") || fail + +case "$ROLE" in + server) bash "${CYCLECLOUD_PROJECT_PATH}/default/files/skel.sh" || fail ;; + login|execute) ;; + *) fail "Unknown pbspro.role '$ROLE'" ;; +esac \ No newline at end of file diff --git a/specs/default/cluster-init/scripts/hwlocs-install.sh b/specs/default/cluster-init/files/hwlocs-install.sh similarity index 65% rename from specs/default/cluster-init/scripts/hwlocs-install.sh rename to specs/default/cluster-init/files/hwlocs-install.sh index 35ec467..d98a2f9 100755 --- a/specs/default/cluster-init/scripts/hwlocs-install.sh +++ b/specs/default/cluster-init/files/hwlocs-install.sh @@ -1,5 +1,8 @@ #!/bin/bash +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 +source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail + PLATFORM_VERSION=$(jetpack props get os.version) || fail if [[ "${PLATFORM_VERSION%%.*}" -ge 8 ]]; then diff --git a/specs/server/cluster-init/scripts/skel.sh b/specs/default/cluster-init/files/skel.sh similarity index 100% rename from specs/server/cluster-init/scripts/skel.sh rename to specs/default/cluster-init/files/skel.sh diff --git a/specs/default/cluster-init/files/utils.sh b/specs/default/cluster-init/files/utils.sh index de46cce..492d45b 100644 --- a/specs/default/cluster-init/files/utils.sh +++ b/specs/default/cluster-init/files/utils.sh @@ -66,4 +66,4 @@ function get_server_hostname() { else echo "$server_hostname" fi -} \ No newline at end of file +} diff --git a/specs/default/cluster-init/install-hook.sh b/specs/default/cluster-init/install-hook.sh new file mode 100644 index 0000000..f7cb00e --- /dev/null +++ b/specs/default/cluster-init/install-hook.sh @@ -0,0 +1,28 @@ +#!/bin/bash +source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 + +ROLE=$(jetpack config pbspro.role "") || fail + +case "$ROLE" in + server) + PACKAGE_TYPE="server" + ROLE_SCRIPT="server-install.sh" + ;; + login) + PACKAGE_TYPE="client" + ROLE_SCRIPT="login-install.sh" + ;; + execute) + PACKAGE_TYPE="execution" + ROLE_SCRIPT="execute-install.sh" + ;; + *) fail "Unknown pbspro.role '$ROLE'" ;; +esac + +bash "${CYCLECLOUD_PROJECT_PATH}/default/files/hwlocs-install.sh" || fail + +PACKAGE_NAME=$(get_package_name "$PACKAGE_TYPE") || fail +jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail +yum install -y -q "/tmp/$PACKAGE_NAME" || fail + +bash "${CYCLECLOUD_PROJECT_PATH}/default/roles/${ROLE_SCRIPT}" || fail diff --git a/specs/execute/cluster-init/stages/install.sh b/specs/default/cluster-init/roles/execute-install.sh similarity index 88% rename from specs/execute/cluster-init/stages/install.sh rename to specs/default/cluster-init/roles/execute-install.sh index 7a501ba..3a086b2 100755 --- a/specs/execute/cluster-init/stages/install.sh +++ b/specs/default/cluster-init/roles/execute-install.sh @@ -1,21 +1,15 @@ #!/bin/bash source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail - -"${CYCLECLOUD_PROJECT_PATH}/default/scripts/hwlocs-install.sh" || fail EXECUTE_HOSTNAME=$(jetpack config hostname) || fail -PACKAGE_NAME=$(get_package_name "execution") || fail SERVER_HOSTNAME=$(get_server_hostname) || fail + # Forces execute node's hostname to be updated (scalelib is blocked until the hostname is correct) # TODO: this installation status should be done by jetpack before cluster-inits are run "${CYCLECLOUD_HOME}/system/embedded/bin/python" -c "import jetpack.converge as jc; jc._send_installation_status('warning')" -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail - if [[ -n "$SERVER_HOSTNAME" ]]; then echo "$SERVER_HOSTNAME" > /var/spool/pbs/server_name chmod 0644 /var/spool/pbs/server_name || fail diff --git a/specs/login/cluster-init/stages/install.sh b/specs/default/cluster-init/roles/login-install.sh similarity index 58% rename from specs/login/cluster-init/stages/install.sh rename to specs/default/cluster-init/roles/login-install.sh index 9840895..f156fed 100755 --- a/specs/login/cluster-init/stages/install.sh +++ b/specs/default/cluster-init/roles/login-install.sh @@ -1,16 +1,9 @@ #!/bin/bash source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 -source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail -"${CYCLECLOUD_PROJECT_PATH}/default/scripts/hwlocs-install.sh" || fail - -PACKAGE_NAME=$(get_package_name "client") || fail SERVER_HOSTNAME=$(get_server_hostname) || fail -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail - if [[ -n "$SERVER_HOSTNAME" ]]; then sed -e "s|__SERVERNAME__|${SERVER_HOSTNAME}|g" \ "${CYCLECLOUD_PROJECT_PATH}/default/templates/default/pbs.conf.template" > /etc/pbs.conf || fail diff --git a/specs/server/cluster-init/stages/install.sh b/specs/default/cluster-init/roles/server-install.sh similarity index 87% rename from specs/server/cluster-init/stages/install.sh rename to specs/default/cluster-init/roles/server-install.sh index 1b45e92..4b03ec0 100644 --- a/specs/server/cluster-init/stages/install.sh +++ b/specs/default/cluster-init/roles/server-install.sh @@ -2,10 +2,6 @@ source "${CYCLECLOUD_PROJECT_PATH}/default/files/utils.sh" || exit 1 source "${CYCLECLOUD_PROJECT_PATH}/default/files/default.sh" || fail - -"${CYCLECLOUD_PROJECT_PATH}/default/scripts/hwlocs-install.sh" || fail - -PACKAGE_NAME=$(get_package_name "server") || fail CLUSTER_NAME=$(jq -r .cluster "$CONFIG_PATH") || fail CONNECTION_URL=$(jq -r .url "$CONFIG_PATH") || fail IGNORE_WORKQ=$(jetpack config pbspro.queues.workq.ignore "False") || fail @@ -13,6 +9,7 @@ IGNORE_HTCQ=$(jetpack config pbspro.queues.htcq.ignore "False") || fail CRON_METHOD=$(jetpack config pbspro.cron_method "pbs_cron") || fail PBSPRO_AUTOSCALE_PROJECT_HOME="/opt/cycle/pbspro" PBSPRO_AUTOSCALE_INSTALLER="cyclecloud-pbspro-pkg-${PBSPRO_AUTOSCALE_VERSION}.tar.gz" +PACKAGE_NAME=$(get_package_name "server") || fail mkdir -p "/sched/${CLUSTER_NAME}" || fail @@ -24,9 +21,6 @@ PBS_SCHEDULER_IP=$(hostname -i) EOF chmod a+r "/sched/${CLUSTER_NAME}/azpbs.env" || fail -jetpack download --project pbspro "$PACKAGE_NAME" "/tmp" || fail -yum install -y -q "/tmp/$PACKAGE_NAME" || fail - mkdir -p -m 0755 "$PBSPRO_AUTOSCALE_PROJECT_HOME" || fail mkdir -p -m 750 /var/spool/pbs/sched_priv || fail @@ -70,10 +64,8 @@ fi ./initialize_default_queues.sh || fail ./install.sh --install-python3 --venv "${INSTALLDIR}/venv" --cron-method "$CRON_METHOD" || fail - +#TODO: modify readme to get this ./generate_autoscale_json.sh --install-dir "$INSTALLDIR" \ - --username "$(jetpack config cyclecloud.config.username)" \ - --password "$(jetpack config cyclecloud.config.password)" \ --url "$CONNECTION_URL" \ --cluster-name "$CLUSTER_NAME" \ $IGNORE_QUEUES_ARG \ diff --git a/specs/execute/cluster-init/stages/activate.sh b/specs/execute/cluster-init/stages/activate.sh deleted file mode 100755 index ba338ca..0000000 --- a/specs/execute/cluster-init/stages/activate.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -systemctl start pbs || exit 1 \ No newline at end of file diff --git a/specs/execute/cluster-init/tests/test_execute.py b/specs/execute/cluster-init/tests/test_execute.py deleted file mode 100644 index 335684d..0000000 --- a/specs/execute/cluster-init/tests/test_execute.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -import unittest -import subprocess - - -class TestExecute(unittest.TestCase): - - def test_simple(self): - p = subprocess.Popen(['/opt/pbs/bin/qstat'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - if hasattr(stdout, "decode"): - stdout = stdout.decode() - stderr = stderr.decode() - - self.assertEqual(0, p.returncode, msg="Call to qstat failed with Stderr: %s\nStdout%s" - % (stderr, stdout)) - diff --git a/specs/login/cluster-init/tests/test_execute.py b/specs/login/cluster-init/tests/test_execute.py deleted file mode 100644 index 335684d..0000000 --- a/specs/login/cluster-init/tests/test_execute.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -import unittest -import subprocess - - -class TestExecute(unittest.TestCase): - - def test_simple(self): - p = subprocess.Popen(['/opt/pbs/bin/qstat'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - if hasattr(stdout, "decode"): - stdout = stdout.decode() - stderr = stderr.decode() - - self.assertEqual(0, p.returncode, msg="Call to qstat failed with Stderr: %s\nStdout%s" - % (stderr, stdout)) - diff --git a/specs/server/cluster-init/tests/helper.py b/specs/server/cluster-init/tests/helper.py deleted file mode 100644 index 28a0f97..0000000 --- a/specs/server/cluster-init/tests/helper.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -from jetpack import config -import subprocess -import os -import sys -import traceback -from io import StringIO -import logging -import platform - -if platform.system() != 'Windows': - import pwd - - -logger = logging.getLogger(__name__) - - -def get_user_profile(user_name, cwd=None): - '''Get user id, group id, and environment variables for specified user''' - pw_record = pwd.getpwnam(user_name) - user_name = pw_record.pw_name - user_home_dir = pw_record.pw_dir - user_uid = pw_record.pw_uid - user_gid = pw_record.pw_gid - env = os.environ.copy() - env['HOME'] = user_home_dir - env['LOGNAME'] = user_name - env['PWD'] = cwd or user_home_dir - env['USER'] = user_name - return user_uid, user_gid, env - - -def demote_to_user(user_uid, user_gid): - '''Demote current process to given user and group''' - def result(): - os.setgid(user_gid) - os.setuid(user_uid) - return result - - -def sudo_check_call(cmd_args, username, cwd=None, env=None): - uid, gid, user_env = get_user_profile(username, cwd=cwd) - user_env.update(env) - subprocess.check_call(cmd_args, cwd=cwd, env=env, preexec_fn=demote_to_user(uid, gid)) - return True - - -def sudo_check_output(cmd_args, username, cwd=None, env={}): - uid, gid, user_env = get_user_profile(username, cwd=cwd) - user_env.update(env) - return subprocess.check_output(cmd_args, cwd=cwd, env=user_env, preexec_fn=demote_to_user(uid, gid)) - - -def get_chef_role(role_name): - return role_name in config.get('roles', []) - - -def exception_to_str(): - t, e, tb = sys.exc_info() - f = StringIO() - traceback.print_tb(tb, None, f) - stack_trace = f.getvalue() - if e.message == '': - exception_message = getattr(e, 'strerror', '') - else: - exception_message = e.message - message = "Encountered exception of type %s with error message:" % type(e) - - if exception_message != '': - message = message + '\n' + exception_message - message = message + '\n' + 'Stacktrace:\n%s' % stack_trace - return message - - -def count_nodes_by_os(output, operating_system): - nodes = output.strip().split('\n\n') - if len(nodes) < 1: - return 0 - opsys = operating_system.upper() - return len(filter(lambda node: 'OpSys = "%s"' % opsys in node, nodes)) diff --git a/specs/server/cluster-init/tests/test_submit.py b/specs/server/cluster-init/tests/test_submit.py deleted file mode 100644 index 79d2dc4..0000000 --- a/specs/server/cluster-init/tests/test_submit.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# -import json -import logging -import os -from subprocess import check_output, check_call, CalledProcessError -import time -import unittest -import uuid - -import helper -import jetpack -from jetpack import props - - -jetpack.util.setup_logging() - -logger = logging.getLogger() - -CLUSTER_USER = props.get("cyclecloud.owner") - - -def readfile_if_exist(filename): - if not os.path.exists(filename): - return '' - - with open(filename) as f: - content = f.read() - return content - - -def write_job_script(): - uid, gid, _ = helper.get_user_profile(CLUSTER_USER) - job_script = '/shared/home/%s/run_hostname.sh' % CLUSTER_USER - - with open(job_script, 'w') as f: - f.write(''' -#!/bin/bash -e - -cat $PBS_NODEFILE > $1 -sleep 3600 -'''.strip()) - - os.chown(job_script, uid, gid) - os.chmod(job_script, 0x0755) - - return job_script - - -class TestSubmit(unittest.TestCase): - - def setUp(self): - self.userhome = "/shared/home/" + CLUSTER_USER - self.results_dir = os.path.join(self.userhome, "test_submit") - if not os.path.exists(self.results_dir): - uid, gid, _ = helper.get_user_profile(CLUSTER_USER) - os.makedirs(self.results_dir) - os.chown(self.results_dir, uid, gid) - - def test_submissions(self): - if jetpack.config.get("pbspro.skip_integration_tests", True): - return - - job_script = write_job_script() - - def submit_job(prefix, qsub_args): - job_uuid = prefix + str(uuid.uuid4()) - output_file_base = self.userhome + '/test_submit/%s' % job_uuid - - helper.sudo_check_output(['/opt/pbs/bin/qsub'] + qsub_args + ["-N", job_uuid, "--", job_script, output_file_base], - CLUSTER_USER, cwd=self.userhome) - return job_uuid - - simple = submit_job("simple-", []) - simple_htcq = submit_job("simple_htcq-", ["-q", "htcq"]) - multipart_select = submit_job("multipart_select-", ["-l", "select=2:ncpus=2+1:ncpus=2"]) - vscatter_excl = submit_job("vscatter_excl=", ["-l", "select=2:ncpus=1", "-l", "place=vscatter:excl"]) - smp = submit_job("smp-", ["-l", "select=2:ncpus=1", "-l", "place=pack:group=host"]) - old_nodes = submit_job("old_nodes-", ["-l", "nodes=2:ppn=2"]) - - # timer that releases the held jobs is set to 15 seconds - time.sleep(30) - - jobs = json.loads(check_output(['qstat', '-f', '-F', 'json'])) - jobs_by_name = {} - for job_id, job in list(jobs.get("Jobs", {}).items()): - job["Job_Id"] = job_id - jobs_by_name[job["Job_Name"]] = job - - def check_job(job_uuid, select, place, slot_type="execute"): - self.assertEqual(select, jobs_by_name[job_uuid]["Resource_List"]["select"]) - self.assertEqual(place, jobs_by_name[job_uuid]["Resource_List"]["place"]) - self.assertEqual(slot_type, jobs_by_name[job_uuid]["Resource_List"]["slot_type"]) - - check_job(simple, "1:ncpus=1:slot_type=execute:ungrouped=false", "pack:group=group_id") - check_job(simple_htcq, "1:ncpus=1:slot_type=execute:ungrouped=true", "pack") - check_job(multipart_select, "2:ncpus=2+1:ncpus=2", "scatter:group=group_id") - check_job(vscatter_excl, "2:ncpus=1", "vscatter:excl:group=group_id") - check_job(smp, "2:ncpus=1", "pack:group=host") - check_job(old_nodes, "2:ncpus=2:mpiprocs=2", "scatter:group=group_id") - - hosts = {} - - def wait_for_results(): - omega = time.time() + 1200 - at_least_one_missing = True - - while at_least_one_missing and time.time() < omega: - try: - # continually collect host information. Some may get shutdown while we are waiting for other jobs to complete. - nodes = json.loads(check_output(["pbsnodes", "-a", "-F", "json"])) - for hostname, node in list(nodes["nodes"].items()): - hosts[hostname.lower()] = node - except CalledProcessError: - # pbsnodes exits with 1 if there are no nodes, just ignore. - pass - - at_least_one_missing = False - result = {} - for job_id in [simple, simple_htcq, multipart_select, vscatter_excl, smp, old_nodes]: - path = os.path.join(self.results_dir, job_id) - if not os.path.exists(path): - at_least_one_missing = True - logger.warn("%s does not exist yet." % path) - else: - with open(path) as fr: - result[job_id] = result.get(job_id, []) - for line in fr.read().splitlines(): - hostname_short = line.strip().split(".")[0].lower() - if hostname_short: - result[job_id].append(hostname_short) - - time.sleep(5) - - return result - - result = wait_for_results() - - def check_hosts(job_uuid, expected_procs, expected_hosts, placed): - self.assertIn(job_uuid, result, "Job %s did not complete" % job_uuid) - self.assertEqual(expected_procs, len(result[job_uuid])) - - # for some types of jobs, there is no guarantee on how many hosts they will land on. - if expected_hosts > 0: - self.assertEqual(expected_hosts, len(set(result[job_uuid]))) - - for hostname in result[job_uuid]: - hostname = hostname.lower() - if placed: - self.assertIsNotNone(hosts[hostname]["resources_available"].get("group_id")) - else: - self.assertIsNone(hosts[hostname]["resources_available"].get("group_id")) - #self.assertEquals(str(not placed).lower(), str(hosts[hostname]["resources_available"]["ungrouped"]).lower()) - - check_hosts(simple, 1, 1, True) - check_hosts(simple_htcq, 1, 1, False) - # -l place=scatter means best effort to scatter, but the jobs could land on a single machine. - check_hosts(multipart_select, 3, -1, True) - check_hosts(vscatter_excl, 2, 2, True) - check_hosts(smp, 2, 1, True) - # 4 mpi procs means 4 processes, so we will see 4 entries in the list but only 2 machines - check_hosts(old_nodes, 4, 2, True) - - # now terminate the sleep jobs so that scale down can happen. - # note, this ensures we scaled at least as many nodes as required. - # if a node was scaled up that wasn't needed, it will fail the scale down assertions in labrat - for job_name in [simple, simple_htcq, multipart_select, vscatter_excl, smp, old_nodes]: - job_id = jobs_by_name[job_name]["Job_Id"] - check_call(["qdel", job_id]) - - -if __name__ == "__main__": - unittest.main() diff --git a/specs/server/cluster-init/tests/tryme.py b/specs/server/cluster-init/tests/tryme.py deleted file mode 100644 index cfd0d0e..0000000 --- a/specs/server/cluster-init/tests/tryme.py +++ /dev/null @@ -1,868 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Bryan W. Berry -# License: BSD New, see LICENSE for details. - -import sys -import os -import time -from abc import ABCMeta, abstractmethod -from functools import wraps, total_ordering - -# Is this Python 3? -PY3 = sys.version_info > (3, 0) - - -class Null(object): - """Null represents nothing.""" - # pylint: disable = too-few-public-methods - def __repr__(self): - return 'Null' - - def __call__(self): - return self - - -# pylint: disable = invalid-name -#: The Null object. -Null = Null() - - -class Unit(object): - """Descriptor that always return the owner monad, used for ``unit``.""" - # pylint: disable = too-few-public-methods - def __get__(self, instance, cls): - """Returns the owner monad.""" - return cls - - -@total_ordering -class Ord(object): - """Mixin class that implements rich comparison ordering methods.""" - # pylint: disable = too-few-public-methods - def __eq__(self, other): - if self is other: - return True - elif not isinstance(other, type(self)): - return NotImplemented - else: - return self._value == other._value - - def __lt__(self, other): - if self is other: - return False - elif isinstance(other, type(self)): - return self._value < other._value - else: - fmt = "unorderable types: {} and {}'".format - raise TypeError(fmt(type(self), type(other))) - - -class Monad(object): - """The Monad Class. - - This is just a base class - """ - __metaclass__ = ABCMeta - - def __init__(self, value): - self._value = value - - def __repr__(self): - return '{cls}({value})'.format( - cls=type(self).__name__, value=repr(self._value)) - - @abstractmethod - def map(self, function): - """The map operation. - - ``function`` is a function that maps from the underlying value to a - monadic type, something like signature ``f :: a -> M a`` in haskell's - term. - """ - return NotImplemented - - #: The ``unit`` of monad. - unit = Unit() - - -class NoSuchElementError(Exception): - pass - - -class FailureError(Exception): - pass - - -class InvalidTryError(Exception): - pass - - -class Try(Monad, Ord): - """A wrapper for operations that may fail - - Represents values/computations with two possibilities. - - :param value: value to contain - :param message: (optional) message to output to console if to_console is called. - If None, a string representation of the contained value is output. - Defaults to ``None`` - :param start: (optional) start time for the operation, in seconds since the UNIX epoch - `time.time()` is typically used for this value. Defaults to ``None``. - :type start: int - :param end: (optional) end time for the operation, in seconds since the UNIX epoch - ``time.time()`` is typically used for this value. Defaults to ``None``. - :type end: int - :param count: (optional) number of times the operation has been executed. Defaults to ``1`` - :type end: int - - Usage:: - - >>> Success(42) - Success(42) - >>> Success([1, 2, 3]) - Success([1, 2, 3]) - >>> Failure('Error') - Failure('Error') - >>> Success(Failure('Error')) - Success(Failure('Error')) - >>> isinstance(Success(1), Try) - True - >>> isinstance(Failure(None), Try) - True - >>> saving = 100 - >>> insolvent = Failure('I am insolvent') - >>> spend = lambda cost: insolvent if cost > saving else Success(saving - cost) - >>> spend(90) - Success(10) - >>> spend(120) - Failure('I am insolvent') - - Map operation with ``map``, applies function to value only if it is a Success and returns a Success - - >>> inc = lambda n: n + 1 - >>> Success(0) - Success(0) - >>> Success(0).map(inc) - Success(1) - >>> Success(0).map(inc).map(inc) - Success(2) - >>> Failure(0).map(inc) - Failure(0) - - Comparison with ``==``, as long as they are the same type and what's - wrapped inside are comparable. - - >>> Failure(42) == Failure(42) - True - >>> Success(42) == Success(42) - True - >>> Failure(42) == Success(42) - False - - A :py:class:`Failure` is less than a :py:class:`Success`, or compare the two by - the values inside if thay are of the same type. - - >>> Failure(42) < Success(42) - True - >>> Success(0) > Failure(100) - True - >>> Failure('Error message') > Success(42) - False - >>> Failure(100) > Failure(42) - True - >>> Success(-2) < Success(-1) - True - """ - def __init__(self, value, message=None, start=None, end=None, count=1): - super(Try, self).__init__(value) - self._message = message - self._start = start - self._end = end - self._count = count - if (start is None and end is not None) or (end is None and start is not None): - raise InvalidTryError( - "The start and end argument must either be both None or not None") - - if type(self) is Try: - raise NotImplementedError('Please use Failure or Success instead') - - def map(self, function): - """The map operation of :py:class:`Try` to Success instances - - Applies function to the value if and only if this is a - :py:class:`Success`. - """ - constructor = type(self) - if self.succeeded(): - return constructor(function(self._value)) - else: - return self - - def map_failure(self, function): - """The map operation of :py:class:`Try` to Failure instances - - Applies function to the value if and only if this is a - :py:class:`Success`. - """ - constructor = type(self) - if self.failed(): - return constructor(function(self._value)) - else: - return self - - def get(self): - '''Gets the Success value if this is a Success otherwise throws an exception''' - if self.succeeded(): - return self._value - else: - raise NoSuchElementError('You cannot call `get` on a Failure, use `get_failure` instead') - - def get_failure(self): - '''Gets the Failure value if this is a Failure otherwise throws an exception''' - if self.failed(): - return self._value - else: - raise NoSuchElementError('You cannot call `get_failure` on a Success, use `get` instead') - - def get_or_else(self, default): - '''Returns the value from this Success or the given default argument if this is a Failure.''' - if self.succeeded(): - return self._value - else: - return default - - def succeeded(self): - """Return a Boolean that indicates if the value is an instance of Success - - >>> Success(True).succeeded() - True - >>> Failure('fubar').succeeded() - False - """ - return bool(self) - - def failed(self): - """Return a Boolean that indicates if the value is an instance of Failure - - >>> Failure('shit is fucked up').failed() - True - >>> Success('it worked!').failed() - False - """ - return not(bool(self)) - - @property - def message(self): - ''' - Return the message for the Try. If the ``message`` argument was provided to the constructor - that value is returned. Otherwise the string representation of the contained value is returened - ''' - if self._message is not None: - return self._message - else: - return str(self._value) - - @property - def start(self): - ''' - Start time of the operation in seconds since the UNIX epoch if specified in - the constructor or with the ``update`` method, ``None`` otherwise - ''' - return self._start - - @property - def end(self): - ''' - End time of the operation in seconds since the UNIX epoch if specified in - the constructor or with the ``update`` method, ``None`` otherwise - ''' - return self._end - - @property - def elapsed(self): - ''' - End time of the operation in seconds since the UNIX epoch if the start and end arguments - were specified in the constructor or with the ``update`` method, ``None`` otherwise - ''' - if self._end is None and self._start is None: - return None - - return self.end - self.start - - @property - def count(self): - '''Number of times the operation has been tried''' - return self._count - - def update(self, message=None, start=None, end=None, count=1): - ''' - Update the Try with new properties but the same value. Returns a new :class:`Failure` - or :py:class:`Success` and does not actually update in place. - - :param message: (optional) message to output to console if to_console is called. - If None, a string representation of the contained value is output. - Defaults to ``None`` - :param start: (optional) start time for the operation, in seconds since the UNIX epoch - `time.time()` is typically used for this value. Defaults to ``None``. - :type start: int - :param end: (optional) end time for the operation, in seconds since the UNIX epoch - ``time.time()`` is typically used for this value. Defaults to ``None``. - :type end: int - :param count: (optional) number of times the operation has been executed. Defaults to ``1`` - :type end: int - ''' - if (start is None and end is not None) or (end is None and start is not None): - raise InvalidTryError( - "The start and end argument must either be both None or not None") - - message = message or self._message - - # start = start or self._start does not work because start may == 0 and is therefore falsey - if start is None: - start = self._start - if end is None: - end = self._end - if count is None: - count = self._count - - constructor = type(self) - return constructor(self._value, message=message, start=start, end=end, count=count) - - def to_console(self, nl=True, exit_err=False, exit_status=1): - ''' - Write a message to the console. By convention, Success messages are written to stdout - and Failure messages are written to stderr. The Failure's `cause` is written to stderr - while the string repesentation of the Success's _value is written. - - :param message: the message to print - :param err: (optional) if set to true the file defaults to ``stderr`` instead of ``stdout``. - :param nl: (optional) if set to `True` (the default) a newline is printed afterwards. - :param exit_err: (optional) if set to True, exit the running program with a non-zero exit code - :param exit_status: (optional) the numeric exist status to return if exit is True - ''' - if self.succeeded(): - to_console(self.message, nl=nl) - else: - to_console(self.message, nl=nl, err=True, exit_err=exit_err, exit_status=exit_status) - - def fail_for_error(self, exit_status=1): - ''' - If a Failure, write the message to stderr and exit with return code of `exit_status` - Does nothing if a Success - - :param exit_status: (optional) the numeric exist status to return if exit is True - :type exit_status: int - ''' - if self.failed(): - to_console(self.message, nl=True, err=True, exit_err=True, - exit_status=exit_status) - - def raise_for_error(self, exception=FailureError): - ''' - Raise an exception if self is an instance of Failure. If the wrapped value is an - instance of Exeception or one of its subclasses, it is raised directly. The the - optional argument ``exception`` is specified, that type is raised with the wrapped - value as its argument. Otherwise, FailureError is raised. This method has no effect - is self is an instance of Success. - - :param exception: (optional) type of Exception to raise - ''' - - if self.succeeded(): - return - - wrapped_value = self.get_failure() - if isinstance(wrapped_value, Exception): - raise wrapped_value - else: - raise exception(wrapped_value) - - def filter(self, predicate): - ''' - If a Success, convert this to a Failure if the predicate is not satisfied. - Applies predicate to the wrapped value - - :param predicate: a function that takes the wrapped value as its argument and returns a boolean value - :rtype: :class:`Try ` object - :return: Try - ''' - if self.failed(): - return self - else: - wrapped_value = self.get() - if predicate(wrapped_value): - return self - else: - return Failure(wrapped_value) - - def __lt__(self, monad): - """Override to handle special case: Success.""" - if not isinstance(monad, (Failure, Success)): - fmt = "unorderable types: {} and {}'".format - raise TypeError(fmt(type(self), type(monad))) - if type(self) is type(monad): - # same type, either both lefts or rights, compare against value - return self._value < monad._value - if monad: - # self is Failure and monad is Success, left is less than right - return True - else: - return False - - def __repr__(self): - """Customize Show.""" - fmt = 'Success({})' if self else 'Failure({})' - return fmt.format(repr(self._value)) - - -class Failure(Try): - """Failure of :py:class:`Try`.""" - def __bool__(self): - # pylint: disable = no-self-use - return False - __nonzero__ = __bool__ - - -class Success(Try): - """Success of :py:class:`Try`.""" - def __bool__(self): - # pylint: disable = no-self-use - return True - - -class Maybe(Monad, Ord): - """A wrapper for values that be None - - >>> Some(42) - Some(42) - >>> Some([1, 2, 3]) - Some([1, 2, 3]) - >>> Some(Nothing) - Some(Nothing) - >>> Some(Some(2)) - Some(Some(2)) - >>> isinstance(Some(1), Maybe) - True - >>> isinstance(Nothing, Maybe) - True - >>> saving = 100 - >>> spend = lambda cost: Nothing if cost > saving else Some(saving - cost) - >>> spend(90) - Some(10) - >>> spend(120) - Nothing - >>> safe_div = lambda a, b: Nothing if b == 0 else Some(a / b) - >>> safe_div(12.0, 6) - Some(2.0) - >>> safe_div(12.0, 0) - Nothing - - Map operation with ``map``. Not that map only applies a function if the object is an - instance of Some. In the case of a Some, ``map`` returns the transformed value inside a Some. - No action is taken for a Nothing. - - >>> inc = lambda n: n + 1 - >>> Some(0) - Some(0) - >>> Some(0).map(inc) - Some(1) - >>> Some(0).map(inc).map(inc) - Some(2) - >>> Nothing.map(inc) - Nothing - - Comparison with ``==``, as long as what's wrapped inside are comparable. - - >>> Some(42) == Some(42) - True - >>> Some(42) == Nothing - False - >>> Nothing == Nothing - True - """ - @classmethod - def from_value(cls, value): - """Wraps ``value`` in a :class:`Maybe` monad. - - Returns a :class:`Some` if the value is evaluated as true. - :data:`Nothing` otherwise. - """ - return cls.unit(value) if value else Nothing - - def get(self): - '''Return the wrapped value if this is Some otherwise throws an exception''' - if self.is_empty(): - raise NoSuchElementError('You cannot call `get` on Nothing') - else: - return self._value - - def get_or_else(self, default): - '''Returns the value from this Some or the given default argument otherwise.''' - if self.is_empty(): - return default - else: - return self._value - - def filter(self, predicate): - ''' - Returns Some(value) if this is a Some and the value satisfies the given predicate. - - :param predicate: a function that takes the wrapped value as its argument and returns a boolean value - :rtype: :class:`Maybe ` object - :return: Maybe - ''' - if self.is_empty(): - return self - else: - wrapped_value = self.get() - if predicate(wrapped_value): - return self - else: - return Nothing - - def map(self, function): - """The map operation of :class:`Maybe`. - - Applies function to the value if and only if this is a :class:`Some`. - """ - constructor = type(self) - return self and constructor(function(self._value)) - - def is_empty(self): - '''Returns true, if this is None, otherwise false, if this is Some.''' - return self is Nothing - - def is_defined(self): - '''Returns true, if this is Some, otherwise false, if this is Nothing.''' - return self is not Nothing - - def __bool__(self): - return self is not Nothing - - __nonzero__ = __bool__ - - def __repr__(self): - """Customized Show.""" - if self is Nothing: - return 'Nothing' - else: - return 'Some({})'.format(repr(self._value)) - - def __iter__(self): - if self is not Nothing: - yield self._value - - -# pylint: disable = invalid-name -Some = Maybe -#: The :class:`Maybe` that represents nothing, a singleton, like ``None``. -Nothing = Maybe(Null) -Maybe.zero = Nothing -# pylint: enable = invalid-name - - -def _get_stacktrace(): - import traceback - if PY3: - from io import StringIO - else: - from StringIO import StringIO - - t, _, tb = sys.exc_info() - f = StringIO() - traceback.print_tb(tb, None, f) - stacktrace = f.getvalue() - return stacktrace - - -def try_out(callable, exception=None): - ''' - Executes a callable and wraps a raised exception in a Failure class. If an exception was - not raised, a Success is returned. If the keyword argument ``exception`` is not None, - only wrap the specified exception. Raise all other exceptions. - The stacktrace related to the exception is added to the wrapped exception - as the `stracktrace` property - - - :param callable: A callable reference, should return a value other than None - :rtype Try: a Success or Failure - ''' - - if exception is None: - catch_exception = Exception - else: - catch_exception = exception - - try: - return Success(callable()) - except catch_exception as e: - stacktrace = _get_stacktrace() - e.stacktrace = stacktrace - return Failure(e) - - -def to_console(message=None, nl=True, err=False, exit_err=False, exit_status=1): - ''' - Write a message to the console - - :param message: the message to print - :param err: (optional) if set to true the file defaults to ``stderr`` instead of ``stdout``. - :param nl: (optional) if set to `True` (the default) a newline is printed afterwards. - :param exit_err: (optional) if set to True, exit the running program with a non-zero exit code - :param exit_status: (optional) the numeric exist status to return if exit is True - ''' - if err: - stream = sys.stderr - else: - stream = sys.stdout - - stream.write(message) - - if nl: - stream.write(os.linesep) - - stream.flush() - if exit_err: - sys.exit(exit_status) - - -class SystemClock: - ''' - This is just a wrapper around the built-in time.time that makes it much easier to test - this module by mocking out time itself. - ''' - def __init__(self): - pass - - def time(self): - '''Returns value of current UNIX epoch in seconds''' - return time.time() - - def sleep(self, seconds): - time.sleep(seconds) - - -class StoppedClock: - ''' - This class only exists to make it easier to test retries - ''' - def __init__(self): - self.times = [] - - def set_times(self, times): - '''list of times for the self.time call to return - the times can be a single value or be a value + side effect to trigger - - example: - clock.set_times([100, 200, (300, lambda: Failure("Uh oh!"), 400]) - - the 3rd invocation of clock.time() will return the Failure - - example: - clock.set_times([100, 200, (300, function_with_side_effect), 400]) - - function_with_side_effect will be triggered the 3rd time clock.time() is invoked - ''' - self.times = times - self.current = iter(times) - - def sleep(self, seconds): - '''This sleep doesn't actually sleep, so your tests run quickly!''' - pass - - def time(self): - current_time = next(self.current) - if not isinstance(current_time, tuple): - return current_time - - current_time_val, side_effect = current_time - if isinstance(side_effect, Exception): - raise side_effect - - side_effect() - - return current_time_val - - -class Counter: - '''A simple counter''' - - def __init__(self, initial=0): - self._count = initial - - def increment(self): - self._count = self._count + 1 - - def reset(self): - self._count = 0 - - @property - def count(self): - return self._count - - -def tick_counter(column_limit=80): - - counter = Counter() - - def write_tick(log): - sys.stdout.write('.') - counter.increment() - # if we have reached the max # of columns, write a newline and reset the counter - if counter.count == column_limit: - sys.stdout.write(os.linesep) - counter.reset() - sys.stdout.flush() - - return write_tick - - -_clock = SystemClock() - - -class Again(Failure): - """ - Again of :py:class:`Failure`. - A handy alias of :py:class:`Failure` to indicate that an operation should be retried - """ - pass - - -class Stop(Success): - """ - Stop of :py:class:`Success`. - A handy alias of :py:class:`Success` to indicate that an operation should **not** be retried - """ - pass - - -class InvalidCallableError(Exception): - pass - - -def raise_if_invalid_result(result): - '''Raise InvalidCallableError if the result is not of type Try''' - if not isinstance(result, Try): - raise InvalidCallableError( - "Functions passed as arguments to the retry function must " - "return either tryme.Success, tryme.Failure, or raise an exception") - - -def retry_wrapper(acallable, timeout=300, delay=5, status_callback=None): - - @wraps(acallable) - def _retry(*args, **kwargs): - start = _clock.time() - assert timeout > 0, 'the timeout keyword argument must be greater than 0' - deadline = start + timeout - counter = Counter(0) - current_time = start - - while current_time < deadline: - counter.increment() - result = acallable(*args, **kwargs) - current_time = _clock.time() - end = current_time - raise_if_invalid_result(result) - - # update with time accounting - result = result.update(start=start, end=end, count=counter.count) - if result.succeeded(): - if status_callback: - status_callback(result) - return result - else: - if status_callback: - status_callback(result) - _clock.sleep(delay) - - return result.update(start=start, end=end, count=counter.count) - - return _retry - - -def retry(*args, **kwargs): - ''' - Function that wraps a callable with a retry loop. The callable should only return - :class:Failure, :class:Success, or raise an exception. This function can - be used as a decorator or directly wrap a function. This method returns a - a result object which is an instance of py:class:`Success` or py:class:`Failure`. - This function updates the result with the time of the first attempt, the time - of the last attempt, and the total count of attempts - - :param acallable: object that can be called - :type acallable: function - :param timeout: (optional) maximum period, in seconds, to wait until an individual try succeeds. - Defaults to ``300`` seconds - :type timeout: int - :param delay: (optional) delay between retries in seconds. Defaults to ``5`` seconds. - :type delay: int - :param status_callback: (optional) callback to invoke after each retry, is passed the result - as an argument. Defaults to ``None``. - :type status_callback: function - - Usage:: - >>> deadline = time.time() + 300 - >>> dinner_iterator = iter([False, False, True]) - >>> def dinner_is_ready(): - ... return next(dinner_iterator) - >>> breakfast_iterator = iter([False, False, True]) - >>> def breakfast_is_ready(): - ... return next(breakfast_iterator) - >>> @retry - ... def wait_for_dinner(): - ... if dinner_is_ready(): - ... return Success("Ready!") - ... else: - ... return Failure("not ready yet") - >>> result = wait_for_dinner() # doctest: +SKIP - >>> result # doctest: +SKIP - Success("Ready!") - >>> result.elapsed # doctest: +SKIP - 8 - >>> result.count # doctest: +SKIP - 3 - >>> @retry - ... def wait_for_breakfast(): - ... if breakfast_is_ready(): - ... return Success("Ready!") - ... else: - ... return Failure("not ready yet") - >>> result = wait_for_breakfast() # doctest: +SKIP - Success("Ready!") - >>> result.elapsed # doctest: +SKIP - 8 - >>> result.count # doctest: +SKIP - 3 - - The names py:class:`Success` and py:class:`Failure` do not always - map well to operations that need to be retried. The subclasses - py:class:`Stop` and py:class:`Again` can be more intuitive.:: - >>> breakfast_iterator = iter([False, False, True]) - >>> def breakfast_is_ready(): - ... return next(breakfast_iterator) - >>> @retry - ... def wait_for_breakfast(): - ... if breakfast_is_ready(): - ... return Stop("Ready!") - ... else: - ... return Again("not ready yet") - - ''' - - # if used as a decorator without arguments `@retry`, the first argument is - # is the decorated function - # If used as a decorator with keyword arguments, say `@retry(timeout=900)` - # the args are empty and the decorated function is supplied sometime later - # as the argument to the decorator. Confusing! - acallable = None - if len(args) > 0: - acallable = args[0] - - if acallable is not None: - return retry_wrapper(acallable, **kwargs) - else: - def decorator(func): - return retry_wrapper(func, **kwargs) - - return decorator diff --git a/templates/openpbs.txt b/templates/openpbs.txt index 5303d1d..70f33e3 100644 --- a/templates/openpbs.txt +++ b/templates/openpbs.txt @@ -17,6 +17,10 @@ Autoscale = $Autoscale Region = $Region KeyPairLocation = ~/.ssh/cyclecloud.pem Azure.Identities = $ManagedIdentity + + # Lustre mounts require termination notifications to unmount + EnableTerminateNotification = ${NFSType == "lustre" || NFSSchedType == "lustre" || AdditionalNFSType == "lustre" || EnableTerminateNotification} + TerminateNotificationTimeout = 10m [[[configuration]]] pbspro.version = $PBSVersion @@ -47,23 +51,27 @@ Autoscale = $Autoscale SSD = True [[[configuration cyclecloud.mounts.nfs_shared]]] - type = nfs + type = $NFSType mountpoint = /shared - export_path = $NFSSharedExportPath + export_path = ${ifThenElse(NFSType == "lustre", strcat("tcp:/lustrefs", NFSSharedExportPath), NFSSharedExportPath)} address = $NFSAddress options = $NFSSharedMountOptions + node_name = server [[[configuration cyclecloud.mounts.nfs_sched]]] - type = nfs + type = $NFSSchedType mountpoint = /sched - disabled = $NFSSchedDisable + export_path = ${ifThenElse(NFSSchedType == "lustre", strcat("tcp:/lustrefs", NFSSchedExportPath), NFSSchedExportPath)} + address = $NFSSchedAddress #note + options = $NFSSchedMountOptions + node_name = server [[[configuration cyclecloud.mounts.additional_nfs]]] - disabled = ${AdditionalNAS isnt true} - type = nfs - address = $AdditonalNFSAddress + disabled = ${AdditionalNFS isnt true} + type = $AdditionalNFSType + address = $AdditionalNFSAddress mountpoint = $AdditionalNFSMountPoint - export_path = $AdditionalNFSExportPath + export_path = ${ifThenElse(AdditionalNFSType == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath), AdditionalNFSExportPath)} options = $AdditionalNFSMountOptions @@ -75,11 +83,12 @@ Autoscale = $Autoscale [[[configuration]]] cyclecloud.discoverable=true - cyclecloud.mounts.nfs_sched.disabled = true - cyclecloud.mounts.nfs_shared.disabled = ${NFSType != "External"} + cyclecloud.mounts.nfs_sched.disabled = $UseBuiltinSched + cyclecloud.mounts.nfs_shared.disabled = $UseBuiltinShared pbspro.cron_method = $AzpbsCronMethod pbspro.queues.workq.ignore = ${Autoscale != true} pbspro.queues.htcq.ignore = ${Autoscale != true} + pbspro.role = server [[[cluster-init cyclecloud/pbspro:server]]] @@ -87,34 +96,38 @@ Autoscale = $Autoscale AssociatePublicIpAddress = $UsePublicNetwork [[[volume sched]]] - Size = 1024 + Size = $SchedFilesystemSize SSD = True Mount = builtinsched - Persistent = False + Persistent = True #TODO: does this need to be true + Disabled = ${!UseBuiltinSched} [[[volume shared]]] - Size = ${ifThenElse(NFSType == "Builtin", FilesystemSize, 2)} + Size = $FilesystemSize SSD = True Mount = builtinshared - Persistent = ${NFSType == "Builtin"} + Persistent = True #TODO: does this need to be true + Disabled = ${!UseBuiltinShared} [[[configuration cyclecloud.mounts.builtinsched]]] + disabled = ${!UseBuiltinSched} mountpoint = /sched fs_type = xfs [[[configuration cyclecloud.mounts.builtinshared]]] - disabled = ${NFSType != "Builtin"} + disabled = ${!UseBuiltinShared} mountpoint = /shared fs_type = xfs [[[configuration cyclecloud.exports.builtinsched]]] + disabled = ${!UseBuiltinSched} export_path = /sched options = no_root_squash samba.enabled = false type = nfs [[[configuration cyclecloud.exports.builtinshared]]] - disabled = ${NFSType != "Builtin"} + disabled = ${!UseBuiltinShared} export_path = /shared samba.enabled = false type = nfs @@ -129,6 +142,7 @@ Autoscale = $Autoscale [[[configuration]]] autoscale.enabled = false + pbspro.role = login [[nodearray execute]] MachineType = $ExecuteMachineType @@ -145,6 +159,7 @@ Autoscale = $Autoscale # {"pbspro": {"ignore_queues": ["workq", "htcq"]}} # which is what pbspro.queues.workq.ignore does as well. # autoscale.enabled = $Autoscale + pbspro.role = execute [[[cluster-init cyclecloud/pbspro:execute]]] @@ -232,61 +247,139 @@ Order = 10 [parameters Network Attached Storage] Order = 15 + [[parameters Shared Storage]] + Order = 10 + + [[[parameter About Shared Storage]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

The directories /sched and /shared are network attached mounts and exist on all nodes of the cluster.
+
+ Options for providing these mounts:
+ [Builtin]: The server node is an NFS server that provides the mountpoint to the other nodes of the cluster.
+ [External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server provides the mountpoint.
+ [Azure Managed Lustre]: An Azure Managed Lustre deployment provides the mountpoint.
+

+

Note: the cluster must be terminated for changes to filesystem mounts to take effect.

''' + Conditions.Hidden := false [[parameters Scheduler Mount]] - Order = 5 + Order = 20 + Label = File-system Mount for /sched + [[[parameter About sched]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template = '''

The directory /sched is a network attached mount and exists in all nodes of the cluster. - It's managed by the scheduler node. - To disable the mount of the /sched directory, and to supply your own for a hybrid scenario, select the checkbox below.''' + Config.Template = '''

OpenPBS configuration shared with cluster nodes is stored in the /sched directory. It is managed by the server node.

''' Order = 6 - [[[parameter NFSSchedDisable]]] + [[[parameter About sched part 2]]] HideLabel = true - DefaultValue = false - Widget.Plugin = pico.form.BooleanCheckBox - Widget.Label = External Scheduler + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Uncheck the box below to disable the built-in NFS export of the /sched directory and use an external file-system.

''' + Order = 7 - [[parameters Default NFS Share]] - Order = 10 - [[[parameter About shared]]] + [[[parameter UseBuiltinSched]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /sched + DefaultValue = true + ParameterType = Boolean + + [[[parameter NFSSchedDiskWarning]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

The directory /shared is a network attached mount and exists in all nodes of the cluster. Users' home directories reside within this mountpoint with the base homedir /shared/home.

There are two options for providing this mount:
[Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster.
[External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server, provides the mountpoint.

Note: the cluster must be terminated for this to take effect.

" - Order = 20 + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinSched - [[[parameter NFSType]]] - Label = NFS Type + [[[parameter NFSSchedType]]] + Label = FS Type ParameterType = StringList - Config.Label = Type of NFS to use for this cluster + Config.Label = Type of shared filesystem to use for this cluster Config.Plugin = pico.form.Dropdown - Config.Entries := {[Label="External NFS"; Value="External"], [Label="Builtin"; Value="Builtin"]} - DefaultValue = Builtin + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedAddress]]] + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedExportPath]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /sched + Conditions.Hidden := UseBuiltinSched + + [[[parameter NFSSchedMountOptions]]] + Label = Mount Options + Description = File System Client Mount Options + Conditions.Hidden := UseBuiltinSched + + [[[parameter SchedFilesystemSize]]] + Label = Size (GB) + Description = The filesystem size (cannot be changed after initial start) + DefaultValue = 1024 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := !UseBuiltinSched + + [[parameters Default NFS Share]] + Order = 30 + Label = File-system Mount for /shared + + [[[parameter About shared]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Users' home directories reside within the /shared mountpoint with the base homedir /shared/home.

''' + Order = 6 + + [[[parameter About shared part 2]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Uncheck the box below to disable the built-in NFS export of the /shared directory and use an external file-system.

''' + Order = 7 + + [[[parameter UseBuiltinShared]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /shared + DefaultValue = true + ParameterType = Boolean [[[parameter NFSDiskWarning]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Warning: switching an active cluster over to NFS will delete the shared disk.

" - Conditions.Hidden := NFSType != "External" + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinShared + + [[[parameter NFSType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Type of shared filesystem to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinShared [[[parameter NFSAddress]]] - Label = NFS IP Address - Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String - Conditions.Hidden := NFSType != "External" + Conditions.Hidden := UseBuiltinShared [[[parameter NFSSharedExportPath]]] - Label = Shared Export Path + Label = Export Path Description = The path exported by the file system DefaultValue = /shared - Conditions.Hidden := NFSType != "External" + Conditions.Hidden := UseBuiltinShared [[[parameter NFSSharedMountOptions]]] - Label = NFS Mount Options - Description = NFS Client Mount Options - Conditions.Hidden := NFSType != "External" + Label = Mount Options + Description = File system client mount options + Conditions.Hidden := UseBuiltinShared [[[parameter FilesystemSize]]] Label = Size (GB) @@ -297,44 +390,56 @@ Order = 15 Config.MinValue = 10 Config.MaxValue = 10240 Config.IntegerOnly = true - Conditions.Excluded := NFSType != "Builtin" + Conditions.Excluded := !UseBuiltinShared [[parameters Additional NFS Mount]] - Order = 20 - [[[parameter Additional NFS Mount Readme]]] + Order = 40 + Label = Additional Filesystem Mount + + [[[parameter Additional Shared FS Mount Readme]]] HideLabel = true Config.Plugin = pico.widget.HtmlTemplateWidget - Config.Template := "

Mount another NFS endpoint on the cluster nodes.

" + Config.Template := "

Mount another shared file-system endpoint on the cluster nodes.

" Order = 20 - [[[parameter AdditionalNAS]]] + [[[parameter AdditionalNFS]]] HideLabel = true DefaultValue = false Widget.Plugin = pico.form.BooleanCheckBox - Widget.Label = Add NFS mount + Widget.Label = Add Shared Filesystem mount - [[[parameter AdditonalNFSAddress]]] - Label = NFS IP Address - Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + [[[parameter AdditionalNFSType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Shared filesystem type of the additional mount + Config.Plugin = pico.form.Dropdown + + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Excluded := AdditionalNFS isnt true + + [[[parameter AdditionalNFSAddress]]] + Label = IP Address + Description = The IP address or hostname of the additional mount. Also accepts a list of comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. Config.ParameterType = String - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSMountPoint]]] - Label = NFS Mount Point + Label = Mount Point Description = The path at which to mount the Filesystem DefaultValue = /data - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSExportPath]]] - Label = NFS Export Path + Label = Export Path Description = The path exported by the file system DefaultValue = /data - Conditions.Excluded := AdditionalNAS isnt true + Conditions.Excluded := AdditionalNFS isnt true [[[parameter AdditionalNFSMountOptions]]] - Label = NFS Mount Options - Description = NFS Client Mount Options - Conditions.Excluded := AdditionalNAS isnt true + Label = Mount Options + Description = Filesystem Client Mount Options + Conditions.Excluded := AdditionalNFS isnt true @@ -364,6 +469,10 @@ Order = 20 Config.Increment = 64 DefaultValue = 0 + [[[parameter EnableTerminateNotification]]] + Label = Enable Termination notifications + DefaultValue = False + [[parameters Software]] diff --git a/util.py b/util.py index d5ef50c..b502a2b 100644 --- a/util.py +++ b/util.py @@ -10,7 +10,7 @@ def download_release_files(): blobs = get_blobs() for _, fname in enumerate(blobs): - if fname == "cyclecloud_api": + if "cyclecloud_api" in fname: continue url = os.path.join(RELEASE_URL, fname) diff --git a/util/Dockerfile b/util/Dockerfile new file mode 100644 index 0000000..d4cf5e0 --- /dev/null +++ b/util/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.11 +RUN pip3.11 install --upgrade setuptools +RUN pip3.11 install --upgrade wheel +RUN pip3.11 install requests \ No newline at end of file diff --git a/util/build.sh b/util/build.sh new file mode 100755 index 0000000..89afb68 --- /dev/null +++ b/util/build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -e +set -x + +if [ "$1" == "-h" ] || [ "$1" == "--help" ] || [ "$1" == "-help" ]; then + echo "Usage: $0 [path/to/scalelib repo]" + echo "If no path to scalelib is passed in, one will be downloaded from GitHub based on" + echo "the version specified in package.py:SCALELIB_VERSION" + exit 1 +fi + +LOCAL_SCALELIB=$1 + +if [ "$LOCAL_SCALELIB" != "" ]; then + LOCAL_SCALELIB=$(realpath $LOCAL_SCALELIB) +fi + +cwd=$(dirname "$(readlink -f "$0")") +SOURCE=$(dirname $cwd) + +if [ ! -e $SOURCE/blobs ]; then + mkdir $SOURCE/blobs +fi + + +cd $SOURCE +rm -f dist/* +./package.sh +mv dist/* blobs/