OSPOOL-158: Add NRP OSPool EP image builds

osg-htc/nrp-ospool-ep/Dockerfile

@@ -0,0 +1,80 @@
+ARG BASE_OSG_SERIES=25
+ARG BASE_YUM_REPO=release
+
+FROM hub.osg-htc.org/osg-htc/ospool-ep:${BASE_OSG_SERIES}-${BASE_YUM_REPO}
+
+# Previous args have gone out of scope
+ARG BASE_OSG_SERIES=25
+ARG BASE_YUM_REPO=release
+
+# Enable OpenCL
+# As suggested by https://github.com/WIPACrepo/pyglidein/blob/master/Dockerfile
+RUN mkdir -p /etc/OpenCL/vendors && \
+    echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+
+# Some helper OpenCL tools
+RUN yum install -y clinfo && \
+    yum clean all
+
+# Fix python vs python3
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
+# Add kubectl, to be able to interact with the k8s cluster
+ARG KUBECTL_VERSION=v1.30.2
+ARG KUBECTL_SHA256=c6e9c45ce3f82c90663e3c30db3b27c167e8b19d83ed4048b61c1013f6a7c66e
+RUN curl -fL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o /usr/sbin/kubectl && \
+    echo "${KUBECTL_SHA256}  /usr/sbin/kubectl" | sha256sum -c - && \
+    chmod u+x /usr/sbin/kubectl
+
+# Tweaked singularity for container environments
+# Remove default singularity from path, too
+COPY scripts/singularity_npid.sh /usr/bin/singularity_npid.sh
+RUN mv /usr/bin/singularity /usr/bin/singularity.org && \
+    chmod a+x /usr/bin/singularity_npid.sh && \
+    ln -s /usr/bin/singularity_npid.sh /usr/bin/singularity
+
+COPY scripts/apptainer_npid.sh /usr/bin/apptainer_npid.sh
+RUN mv /usr/bin/apptainer /usr/bin/apptainer.org && \
+    chmod a+x /usr/bin/apptainer_npid.sh && \
+    ln -s /usr/bin/apptainer_npid.sh /usr/bin/apptainer
+
+# We want to be able to add system-level changes
+# and run without explicit privilege drop
+RUN mv /usr/local/sbin/entrypoint.sh /usr/local/sbin/entrypoint.osg.sh
+COPY scripts/entrypoint.sh /usr/local/sbin/entrypoint.sh
+RUN chmod a+rx /usr/local/sbin/entrypoint.sh
+
+COPY scripts/check_master.sh /bin/check_master.sh
+RUN chmod a+rx /bin/check_master.sh
+
+# system level scripts
+COPY scripts/01_token.sh /etc/entrypoint/image-config.d/01_token.sh
+COPY scripts/01_no_condor_host.sh /etc/entrypoint/image-config.d/01_no_condor_host.sh
+COPY scripts/11_set_OSGInstitutionID.sh /etc/entrypoint/image-config.d/11_set_OSGInstitutionID.sh
+
+# pilot level scripts
+COPY scripts/02_validate_singularity.sh /etc/osg/image-init.d/02_validate_singularity.sh 
+COPY scripts/02_validate_apptainer.sh /etc/osg/image-init.d/02_validate_apptainer.sh 
+COPY scripts/19_set_resources.sh /etc/osg/image-init.d/19_set_resources.sh
+COPY scripts/20_advertise_glidein.sh /etc/osg/image-init.d/20_advertise_glidein.sh
+COPY scripts/20_advertise_k8s_domain.sh /etc/osg/image-init.d/20_advertise_k8s_domain.sh
+COPY scripts/21_advertise_k8s_provisioner.sh  /etc/osg/image-init.d/21_advertise_k8s_provisioner.sh
+COPY scripts/22_set_requirements.sh /etc/osg/image-init.d/22_set_requirements.sh
+
+# keep default low, as we expect to be running in opportunistic mode
+ENV ACCEPT_JOBS_FOR_HOURS=24
+
+# keep default idle time low, as we may over-provision certain kinds of resources
+# but others may be waiting
+ENV ACCEPT_IDLE_MINUTES=20
+
+#
+# In addition to osgvo-docker-pilot envs,
+# it accepts also the following env variables
+# NUM_CPUS MEMORY DISK NUM_GPUS
+# PHYSICAL_HOSTNAME
+# K8S_NAMESPACE K8S_DOMAIN
+# K8S_PROVISIONER_NAME K8S_PROVISIONER_TYPE
+# FORCE_K8SNAMESPACE_MATCHING ADDITIONAL_REQUIREMENTS
+#
+

osg-htc/nrp-ospool-ep/build-config.json

@@ -0,0 +1,7 @@
+{
+    "standard_build": true,
+    "repo_build": false,
+    "base_os": ["el9"],
+    "osg_series": ["24", "25"],
+    "base_repo": ["release"]
+  }

osg-htc/nrp-ospool-ep/scripts/01_no_condor_host.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+#
+# osgvo pilot does not like it if CONDOR_HOST is set
+#
+
+export CONDOR_HOST=
+unset CONDOR_HOST
+

osg-htc/nrp-ospool-ep/scripts/01_token.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#
+# osgvo-pilot expects the token as an env variable
+#
+
+token_file=/etc/condor/tokens.d/prp-wn.token
+if [ ! -r "$token_file" ]; then
+    echo "ERROR: required token file '$token_file' is missing or unreadable" >&2
+    exit 1
+fi
+
+read TOKEN < "$token_file"
+export TOKEN

osg-htc/nrp-ospool-ep/scripts/02_validate_apptainer.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# check if we need to test nvidia
+nvf=
+ls -l /dev/nvidia*
+if [ $? -eq 0 ]; then
+  nvf=--nv
+fi
+
+# else do nothing, let Condor figure it out
+
+if [ -f "/usr/bin/apptainer" ]; then
+  # only test for apptainer functionality if apptainer is present
+  # may not be in all pods
+
+  /usr/bin/apptainer exec $nvf --contain --ipc --pid --bind /cvmfs /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el7:latest /usr/bin/dc -e "3 5 + p"
+  rc=$?
+
+  if [ $rc -ne 0 ]; then
+    echo "Apptainer test execution failed!"
+    sleep 30
+    exit 1
+  fi
+
+fi

osg-htc/nrp-ospool-ep/scripts/02_validate_singularity.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# check if we need to test nvidia
+nvf=
+ls -l /dev/nvidia*
+if [ $? -eq 0 ]; then
+  nvf=--nv
+fi
+
+# else do nothing, let Condor figure it out
+
+if [ -f "/usr/bin/singularity" ]; then
+  # only test for singularity functionality if singularity is present
+  # may not be in all pods
+
+  /usr/bin/singularity exec $nvf --contain --ipc --pid --bind /cvmfs /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el8:latest /usr/bin/dc -e "3 5 + p"
+  rc=$?
+
+  if [ $rc -ne 0 ]; then
+    echo "Singularity test execution failed!"
+    sleep 30
+    exit 1
+  fi
+
+fi

osg-htc/nrp-ospool-ep/scripts/11_set_OSGInstitutionID.sh

@@ -0,0 +1,10 @@
+#
+# Fetch OSGInstitutionID from k8s and set it as an env variable
+# unless it is already set
+#
+if [ "x${OSG_INSTITUTION_ID}" == "x" ]; then
+   OSG_INSTITUTION_ID=`/usr/sbin/kubectl get node ${PHYSICAL_HOSTNAME} -o jsonpath="{.metadata.labels['nautilus\.io/OSGInstitutionID']}"`
+   if [ "x${OSG_INSTITUTION_ID}" != "x" ]; then
+      export OSG_INSTITUTION_ID
+   fi
+fi

osg-htc/nrp-ospool-ep/scripts/19_set_resources.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+
+#
+# Advertise the k8s namespace and physical hostname
+#
+
+#
+# NUM_CPUS and MEMORY are also handled by the osgvo-pilot
+# but we re-set it here
+#
+
+full_num_cpus="${NUM_CPUS:-1}"
+full_memory="${MEMORY:-1024}"
+full_disk="${DISK:-100000}"
+full_num_gpus="${NUM_GPUS:-0}"
+
+echo "NUM_CPUS = ${full_num_cpus}" >> "${PILOT_CONFIG_FILE}"
+echo "MEMORY = ${full_memory}" >> "${PILOT_CONFIG_FILE}"
+echo "DISK = ${full_disk}" >> "${PILOT_CONFIG_FILE}"
+
+# single slot using all the requested resources
+echo "NUM_SLOTS_TYPE_1 = 1" >> "${PILOT_CONFIG_FILE}"
+echo "SLOT_TYPE_1_PARTITIONABLE = FALSE" >> "${PILOT_CONFIG_FILE}"
+
+if [ "x${full_num_gpus}" != "x0" ]; then
+   # we cannot really set the number of GPUs, just enable auto-detect
+   echo "use feature : GPUs" >> "${PILOT_CONFIG_FILE}"
+   echo "SLOT_TYPE_1 = cpu=${full_num_cpus},mem=${full_memory},disk=auto,swap=auto,gpus=${full_num_gpus}" \
+      >> "${PILOT_CONFIG_FILE}"
+else
+   echo "SLOT_TYPE_1 = cpu=${full_num_cpus},mem=${full_memory},disk=auto,swap=auto" \
+      >> "${PILOT_CONFIG_FILE}"
+fi
+

osg-htc/nrp-ospool-ep/scripts/20_advertise_glidein.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+
+#
+# Advertise that this is a glidein
+#
+
+if [ "x${ADVERTISE_IS_GLIDEIN}" != "xN" ]; then
+  echo "IS_GLIDEIN = true" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) IS_GLIDEIN'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+

osg-htc/nrp-ospool-ep/scripts/20_advertise_k8s_domain.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+
+#
+# Advertise the k8s namespace and physical hostname
+#
+
+echo "# K8S params" >> "${PILOT_CONFIG_FILE}"
+
+if [ "x${HOSTNAME}" != "x" ]; then
+  echo "K8SPodName=\"${HOSTNAME}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SPodName'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+if [ "x${K8S_DOMAIN}" != "x" ]; then
+  echo "K8SDomain=\"${K8S_DOMAIN}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SDomain'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+if [ "x${K8S_NAMESPACE}" != "x" ]; then
+  echo "K8SNamespace=\"${K8S_NAMESPACE}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SNamespace'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+if [ "x${PHYSICAL_HOSTNAME}" != "x" ]; then
+  echo "K8SPhysicalHostName=\"${PHYSICAL_HOSTNAME}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SPhysicalHostName'  >> "${PILOT_CONFIG_FILE}"
+fi
+

osg-htc/nrp-ospool-ep/scripts/21_advertise_k8s_provisioner.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+
+#
+# Advertise the k8s provisioner
+#
+
+if [ "x${K8S_PROVISIONER_TYPE}" != "x" ]; then
+  echo "K8SProvisionerType=\"${K8S_PROVISIONER_TYPE}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SProvisionerType'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+if [ "x${K8S_PROVISIONER_NAME}" != "x" ]; then
+  echo "K8SProvisionerName=\"${K8S_PROVISIONER_NAME}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SProvisionerName'  >> "${PILOT_CONFIG_FILE}"
+fi
+

osg-htc/nrp-ospool-ep/scripts/22_set_requirements.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+cat >>  "${PILOT_CONFIG_FILE}" << EOF
+#
+# Force matching on K8SNamespace
+# unless FORCE_K8SNAMESPACE_MATCHING=="no"
+#
+
+FORCE_K8SNAMESPACE_MATCHING = "${FORCE_K8SNAMESPACE_MATCHING:-no}"
+STARTD_EXPRS = \$(STARTD_EXPRS) FORCE_K8SNAMESPACE_MATCHING
+
+MATCHING_START = ( (FORCE_K8SNAMESPACE_MATCHING=?="no") || regexp(TARGET.RequestK8SNamespace,K8SNamespace) )
+
+#
+# Force matching on provisioned resources
+#
+
+PROVISIONING_START = ifthenelse(TARGET.RequestCPUs=!=undefined, CPUs=?=TARGET.RequestCPUs, CPUs=?=1)
+
+# Pretend small memory and disk requests are the equivalent of not set
+# Note: 4096 is special in the provisioner
+PROVISIONING_START = \$(PROVISIONING_START) && \\
+                     ifthenelse(TARGET.RequestMemory=!=undefined, \\
+                           ifthenelse(TARGET.RequestMemory<4096, \\
+                                 Memory=?=4096, \\
+                                 ((Memory>=TARGET.RequestMemory) && ((TARGET.RequestMemory+1536)>Memory))), \\
+                           Memory=?=4096)
+
+# Note: 8000000 is special in the provisioner
+PROVISIONING_START = \$(PROVISIONING_START) && \\
+                     ifthenelse(TARGET.RequestDisk=!=undefined, \\
+                           ifthenelse(TARGET.RequestDisk<8000000, \\
+                                 Disk=?=8000000, \\
+                                 ((Disk>=TARGET.RequestDisk) && ((TARGET.RequestDisk+4500000)>Disk))), \\
+                           Disk=?=8000000)
+
+# GPUs will not be defined if there are no GPUs
+PROVISIONING_START = \$(PROVISIONING_START) && \\
+                     ifthenelse(TARGET.RequestGPUs=!=undefined, \\
+                                ifthenelse(GPUs=!=undefined, GPUs=?=TARGET.RequestGPUs, TARGET.RequestGPUs=?=0), \\
+                                (GPUs=?=undefined) || (GPUs=?=0))
+
+START = ( \$(START) ) && ( \$(PROVISIONING_START) ) && ( \$(MATCHING_START) )
+
+EOF
+
+
+if [ "x${ADDITIONAL_REQUIREMENTS}" != "x" ]; then
+  echo "# Additional requirements added at runtime " >> "${PILOT_CONFIG_FILE}"
+  echo "MATCHING_START = ( \$(MATCHING_START) ) && ( ${ADDITIONAL_REQUIREMENTS} )" >> "${PILOT_CONFIG_FILE}"
+fi

osg-htc/nrp-ospool-ep/scripts/apptainer_npid.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# simple apptainer wrapper that doesn't allow the -p or --pid option
+# also rewrite -C to remove the -p option
+# Disclaimer: Based on 
+#  https://wiki-dev.bash-hackers.org/scripting/posparams
+#
+
+options=()  # the buffer array for the parameters
+eoo=0       # end of options reached
+
+while [[ $1 ]]
+do
+    if ! ((eoo)); then
+	case "$1" in
+	  --pid)
+              # pretend was not passed
+	      shift
+	      ;;
+          -p)
+              # pretend was not passed
+              shift
+              ;;
+          --containall)
+              options+=("--contain")
+              options+=("--cleanenv")
+              options+=("--ipc")
+              # but not --pid
+              shift
+              ;;
+          -C)
+              options+=("-c")
+              options+=("-e")
+              options+=("-i")
+              # but not -p
+              shift
+              ;;
+	  --)
+	      eoo=1
+	      options+=("$1")
+	      shift
+	      ;;
+	  *)
+	      options+=("$1")
+	      shift
+	      ;;
+	esac
+    else
+	options+=("$1")
+	shift
+    fi
+done
+
+exec /cvmfs/oasis.opensciencegrid.org/mis/apptainer/bin/apptainer "${options[@]}"
+