diff --git a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/Dockerfile.ais b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/Dockerfile.ais new file mode 100644 index 0000000000..99b0bc848c --- /dev/null +++ b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/Dockerfile.ais @@ -0,0 +1,125 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Dockerfile for building an OpenSearch image using UBI9 for ppc64le / OCP compatibility. +# It assumes that the working directory contains these files: an OpenSearch tarball (opensearch-ppc64le.tgz), +# log4j2.properties, opensearch.yml, opensearch-docker-entrypoint.sh, and opensearch-onetime-setup.sh. +# +# Build arguments: +# VERSION: Required. Used to label the image. +# BUILD_DATE: Required. Used to label the image (e.g. 'yyyy-mm-ddThh:mm:ssZ'). +# UID: Optional. Specify the opensearch userid. Defaults to 1000. +# GID: Optional. Specify the opensearch groupid. Defaults to 1000. +# OPENSEARCH_HOME: Optional. Specify the opensearch root directory. Defaults to /usr/share/opensearch. + +########################### Stage 0 ######################## +FROM registry.access.redhat.com/ubi9:latest AS linux_stage_0 + +ARG UID=1000 +ARG GID=1000 +ARG VERSION +ARG TEMP_DIR=/tmp/opensearch +ARG OPENSEARCH_HOME=/usr/share/opensearch +ARG OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config +ARG SECURITY_PLUGIN_DIR=$OPENSEARCH_HOME/plugins/opensearch-security +ARG PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR=$OPENSEARCH_PATH_CONF/opensearch-performance-analyzer + +# Apply CVE mitigations and install required extraction/user tools +RUN dnf upgrade -y curl-minimal libcurl-minimal libpng && \ + dnf install -y tar gzip shadow-utils which openssl && \ + dnf clean all + +# Create an opensearch user and group, add to GID 0 (root) for OpenShift arbitrary UID compatibility +RUN groupadd -g $GID opensearch && \ + adduser -u $UID -g $GID -G 0 -d $OPENSEARCH_HOME opensearch && \ + mkdir -p $TEMP_DIR $OPENSEARCH_HOME + +# Prepare working directory and copy artifacts +COPY * $TEMP_DIR/ +RUN ls -l $TEMP_DIR && \ + tar -xzpf /tmp/opensearch/opensearch-ppc64le.tgz -C $OPENSEARCH_HOME --strip-components=1 && \ + MAJOR_VERSION_ENTRYPOINT=`echo $VERSION | cut -d. -f1` && \ + echo $MAJOR_VERSION_ENTRYPOINT && \ + if ! (ls $TEMP_DIR | grep -E "opensearch-docker-entrypoint-.*.x.sh" | grep $MAJOR_VERSION_ENTRYPOINT); then MAJOR_VERSION_ENTRYPOINT="default"; fi && \ + mkdir -p $OPENSEARCH_HOME/data && chown -Rv $UID:$GID $OPENSEARCH_HOME/data && \ + if [[ -d $SECURITY_PLUGIN_DIR ]] ; then chmod -v 750 $SECURITY_PLUGIN_DIR/tools/* ; fi && \ + if [[ -d $PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR ]] ; then cp -v $TEMP_DIR/performance-analyzer.properties $PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR; fi && \ + cp -v $TEMP_DIR/opensearch-docker-entrypoint-$MAJOR_VERSION_ENTRYPOINT.x.sh $OPENSEARCH_HOME/opensearch-docker-entrypoint.sh && \ + cp -v $TEMP_DIR/opensearch-onetime-setup.sh $OPENSEARCH_HOME/ && \ + cp -v $TEMP_DIR/log4j2.properties $TEMP_DIR/opensearch.yml $OPENSEARCH_PATH_CONF/ && \ + ls -l $OPENSEARCH_HOME && \ + rm -rf $TEMP_DIR + + +########################### Stage 1 ######################## +FROM registry.access.redhat.com/ubi9:latest + +ARG UID=1000 +ARG GID=1000 +ARG OPENSEARCH_HOME=/usr/share/opensearch + +# Install tools, OpenSSL for certificate generation, and Java 21 OpenJDK. +# OpenSearch 3.5.0 requires JDK 21; JDK 25 removes APIs used by 3.5.0 Gradle plugins. +# Note: java-21-openjdk-crypto-adapter is x86_64-only and not available on ppc64le; +# FIPS approved-only mode is disabled anyway via OPENSEARCH_JAVA_OPTS below. +RUN dnf upgrade -y curl-minimal libcurl-minimal libpng && \ + dnf install -y tar gzip shadow-utils which openssl \ + java-21-openjdk \ + java-21-openjdk-devel \ + java-21-openjdk-headless && \ + dnf clean all + +# Point JAVA_HOME and PATH to the system installed OpenJDK (ppc64le has no bundled JDK) +ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk +ENV PATH=$OPENSEARCH_HOME/bin:$JAVA_HOME/bin:$PATH + +# Create opensearch user and add to root group (GID 0) for OpenShift arbitrary UID compatibility +RUN groupadd -g $GID opensearch && \ + adduser -u $UID -g $GID -G 0 -d $OPENSEARCH_HOME opensearch + +# Copy unpacked OpenSearch tree from Stage 0 +COPY --from=linux_stage_0 --chown=$UID:0 $OPENSEARCH_HOME $OPENSEARCH_HOME +WORKDIR $OPENSEARCH_HOME + +# Add k-NN lib directory to library loading path variable +ENV LD_LIBRARY_PATH="$OPENSEARCH_HOME/plugins/opensearch-knn/lib" + +# Disable FIPS approved-only mode to allow standard passwords +ENV OPENSEARCH_JAVA_OPTS="-Dorg.bouncycastle.fips.approved_only=false" + +# Run setup scripts +ARG DISABLE_INSTALL_DEMO_CONFIG=true +ARG DISABLE_SECURITY_PLUGIN=false +RUN ./opensearch-onetime-setup.sh + +# OpenShift restricted-v2 compatibility: +# Set full group ownership (GID 0) and permissions AFTER all setup scripts finish +RUN chown -R $UID:0 $OPENSEARCH_HOME && \ + chmod -R g+rwX $OPENSEARCH_HOME && \ + find $OPENSEARCH_HOME -type d -exec chmod g+x {} \; && \ + chmod g+rwX $OPENSEARCH_HOME/opensearch-docker-entrypoint.sh && \ + chmod g+rwX $OPENSEARCH_HOME/opensearch-onetime-setup.sh + +# Change to unprivileged user +USER $UID + +# Expose ports: 9200 (HTTP), 9300 (Transport), 9600 (Performance Analyzer Agent), 9650 (Root Cause Analysis) +EXPOSE 9200 9300 9600 9650 + +ARG VERSION +ARG BUILD_DATE +ARG NOTES + +# Metadata Labels +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.name="opensearch" \ + org.label-schema.version="$VERSION" \ + org.label-schema.url="https://opensearch.org" \ + org.label-schema.vcs-url="https://github.com/opensearch-project/OpenSearch" \ + org.label-schema.license="Apache-2.0" \ + org.label-schema.vendor="OpenSearch" \ + org.label-schema.description="$NOTES" \ + org.label-schema.build-date="$BUILD_DATE" + +ENTRYPOINT ["./opensearch-docker-entrypoint.sh"] +CMD ["opensearch"] diff --git a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/Dockerfile.build-env b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/Dockerfile.build-env new file mode 100644 index 0000000000..c3d1f861e7 --- /dev/null +++ b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/Dockerfile.build-env @@ -0,0 +1,94 @@ +FROM registry.access.redhat.com/ubi9/ubi + +# Set working directory +WORKDIR /opensearch-build + +# Apply CVE mitigations before installing any other packages. +RUN yum upgrade -y curl-minimal libcurl-minimal libpng && yum clean all + +# Install system dependencies +RUN yum install -y --allowerasing \ + git \ + gcc \ + gcc-c++ \ + gcc-gfortran \ + make \ + patch \ + tar \ + unzip \ + zip \ + which \ + curl \ + jq \ + python3 \ + python3-devel \ + python3-pip \ + bzip2-devel \ + zlib-devel \ + openssl-devel \ + libffi-devel \ + xz-devel \ + rpm-build \ + && yum clean all + +# Install yq v4 (required by assemble scripts) +RUN curl -fsSL "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_ppc64le" \ + -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + +# Install JDK 21 (required by OpenSearch 3.5.0; JDK 25 removes APIs used by 3.5.0 Gradle plugins). +RUN yum install -y \ + java-21-openjdk-devel && \ + yum upgrade -y \ + java-21-openjdk \ + java-21-openjdk-devel \ + java-21-openjdk-headless && \ + yum clean all + +# Set Java environment +ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk +ENV PATH=$JAVA_HOME/bin:$PATH +ENV GRADLE_USER_HOME=/root/.gradle + +# Pre-cache all Gradle distributions used by 3.5.0 components to avoid +# network timeouts during the build (each component's gradlew would otherwise +# download its own distribution at build time). +# Versions needed (verified from gradle-wrapper.properties in each 3.5.0.0 tag): +# 9.2.1 - OpenSearch core (OpenSearch/3.5.0), cross-cluster-replication +# 9.2.0 - all other components: common-utils, job-scheduler, security, +# ml-commons, k-NN, alerting, index-management, neural-search, +# ltr, observability, reporting +RUN for version in 9.2.1 9.2.0; do \ + mkdir -p /root/.gradle/wrapper/dists/gradle-${version}-all && \ + curl -fsSL "https://services.gradle.org/distributions/gradle-${version}-all.zip" \ + -o /tmp/gradle-${version}-all.zip && \ + HASH=$(sha256sum /tmp/gradle-${version}-all.zip | cut -d' ' -f1) && \ + DEST="/root/.gradle/wrapper/dists/gradle-${version}-all/${HASH}" && \ + mkdir -p "${DEST}" && \ + mv /tmp/gradle-${version}-all.zip "${DEST}/gradle-${version}-all.zip" && \ + touch "${DEST}/gradle-${version}-all.zip.ok"; \ + done + +# Install Maven +RUN MAVEN_URL=$(curl -s https://maven.apache.org/download.cgi \ + | grep -Eo '["\047].*.bin.tar.gz["\047]' | tr -d "\"'" | uniq | head -n 1) && \ + mkdir -p /usr/local/apache-maven && \ + curl -s "$MAVEN_URL" | tar xzf - --strip-components=1 -C /usr/local/apache-maven && \ + ln -sfn /usr/local/apache-maven/bin/mvn /usr/local/bin/mvn + +# Install pipenv (pin setuptools+virtualenv to avoid packaging incompatibility with pipenv 2023.6.12) +# Also install PyYAML into the system Python so that inline scripts can import yaml +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install --ignore-installed \ + "setuptools==67.8.0" \ + "virtualenv==20.24.5" \ + "pipenv==2023.6.12" \ + "PyYAML>=6.0.2" + +# Copy the build repo +COPY . . + +# Pre-install Python dependencies +RUN python3 -m pipenv install --deploy --ignore-pipfile + +# Set default command +CMD ["/bin/bash"] diff --git a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/build-image-single-arch.sh b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/build-image-single-arch.sh deleted file mode 100644 index 2ec18f58b1..0000000000 --- a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/build-image-single-arch.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash - -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -# This script is to automate the docker image creation process of OpenSearch and OpenSearch-Dashboards - -set -e - -# Import libs -. ../../lib/shell/file_management.sh - -function usage() { - echo "" - echo "This script is used to build the OpenSearch Docker image with single architecture (x64 or arm64). It prepares the files required by the Dockerfile in a temporary directory, then builds and tags the Docker image." - echo "--------------------------------------------------------------------------" - echo "Usage: $0 [args]" - echo "" - echo "Required arguments:" - echo -e "-v VERSION\tSpecify the OpenSearch version number that you are building, e.g. '1.0.0' or '1.0.0-beta1'. This will be used to label the Docker image. If you do not use the '-o' option then this tool will download a public OPENSEARCH release matching this version." - echo -e "-f DOCKERFILE\tSpecify the dockerfile full path, e.g. dockerfile/opensearch.al2.dockerfile." - echo -e "-p PRODUCT\tSpecify the product, e.g. opensearch or opensearch-dashboards, make sure this is the name of your config folder and the name of your .tgz defined in dockerfile." - echo -e "-a ARCHITECTURE\tSpecify one and only one architecture, e.g. x64 or arm64." - echo "" - echo "Optional arguments:" - echo -e "-t TARBALL\tSpecify a local opensearch or opensearch-dashboards tarball. You still need to specify the version - this tool does not attempt to parse the filename." - echo -e "-n NOTES\tSpecify Pipeline Notes of the run, defaults to None." - echo -e "-h\t\tPrint this message." - echo "--------------------------------------------------------------------------" -} - -while getopts ":ht:n:v:f:p:a:" arg; do - case $arg in - h) - usage - exit 1 - ;; - t) - TARBALL=`realpath $OPTARG` - ;; - n) - NOTES=$OPTARG - ;; - v) - VERSION=$OPTARG - ;; - f) - DOCKERFILE=$OPTARG - ;; - p) - PRODUCT=$OPTARG - ;; - a) - ARCHITECTURE=$OPTARG - ;; - :) - echo "-${OPTARG} requires an argument" - usage - exit 1 - ;; - ?) - echo "Invalid option: -${OPTARG}" - exit 1 - ;; - esac -done - -# Validate the required parameters to present -if [ -z "$VERSION" ] || [ -z "$DOCKERFILE" ] || [ -z "$PRODUCT" ] || [ -z "$ARCHITECTURE" ]; then - echo "You must specify '-v VERSION', '-f DOCKERFILE', '-p PRODUCT', '-a ARCHITECTURE'" - usage - exit 1 -else - echo $VERSION $DOCKERFILE $PRODUCT $ARCHITECTURE -fi - -if [ "$PRODUCT" != "opensearch" ] && [ "$PRODUCT" != "opensearch-dashboards" ] -then - echo "Enter either 'opensearch' or 'opensearch-dashboards' as product name for -p parameter" - exit 1 -else - PRODUCT_ALT=`echo $PRODUCT | sed 's@-@_@g'` - echo $PRODUCT $PRODUCT_ALT.yml -fi - -if [ "$ARCHITECTURE" != "x64" ] && [ "$ARCHITECTURE" != "arm64" ] && [ "$ARCHITECTURE" != "ppc64le" ] -then - echo "We only support 'x64' and 'arm64' as architecture name for -a parameter" - exit 1 -fi - -if [ -z "$NOTES" ] -then - NOTES="None" -fi - -# Create temp workdirectory -DIR=`Temp_Folder_Create` -Trap_File_Delete_No_Sigchld $DIR -echo New workspace $DIR - -# Copy configs -cp -v config/${PRODUCT}/* $DIR/ -cp -v ../../config/${PRODUCT_ALT}*.yml $DIR/ -cp -v ../../scripts/opensearch-onetime-setup.sh $DIR/ -cp -v dockerfiles/opensearch-knn-3.5.0.0.zip $DIR/ -cp -v dockerfiles/opensearch-ml-3.5.0.0-SNAPSHOT.zip $DIR/ -cp -v dockerfiles/opensearch-neural-search-3.5.0.0-SNAPSHOT.zip $DIR/ -cp -v dockerfiles/opensearch-security-3.5.0.0.zip $DIR/ -cp -v dockerfiles/opensearch-job-scheduler-3.5.0.0-SNAPSHOT.zip $DIR/ -cp -v dockerfiles/opensearch-alerting-3.5.0.0-SNAPSHOT.zip $DIR/ -cp -r dockerfiles/opensearch-cross-cluster-replication-3.5.0.0-SNAPSHOT.zip $DIR/ -cp -v dockerfiles/Dockerfile $DIR - -# Copy TGZ -arch_uname=`echo ${ARCHITECTURE} | sed 's/x64/x86_64/g;s/arm64/aarch64/g'` -if [ -z "$TARBALL" ]; then - # No tarball file specified so use local build - LOCAL_TAR="./opensearch-min-3.5.0-linux-ppc64le.tar.gz" - if [ -f "$LOCAL_TAR" ]; then - echo -e "\nUsing locally built ${PRODUCT} arch ${ARCHITECTURE} version ${VERSION} from ${LOCAL_TAR}" - cp -v $LOCAL_TAR $DIR/opensearch-core-ppc64le.tgz - file $DIR/opensearch-core-ppc64le.tgz - ls -l $DIR - else - echo "Error: Local tarball not found at ${LOCAL_TAR}" - echo "Please ensure opensearch-min-3.5.0-linux-ppc64le.tar.gz exists in the same directory as this script" - exit 1 - fi -else - echo -e "\nCopying ${PRODUCT} arch ${ARCHITECTURE} version ${VERSION} from ${TARBALL}" - cp -v $TARBALL $DIR/opensearch-core-ppc64le.tgz - file $DIR/opensearch-core-ppc64le.tgz - ls -l $DIR -fi - -# Docker build -docker build --no-cache --build-arg VERSION=$VERSION --build-arg BUILD_DATE=`date -u +%Y-%m-%dT%H:%M:%SZ` --build-arg NOTES=$NOTES --secret id=artifactory_user_secret,env=artifactory_user --secret id=artifactory_token_secret,env=artifactory_token -f $DOCKERFILE $DIR -t opensearchproject/$PRODUCT:$VERSION -docker tag opensearchproject/$PRODUCT:$VERSION opensearchproject/$PRODUCT:latest diff --git a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/build.sh b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/build.sh new file mode 100755 index 0000000000..00dcbf5788 --- /dev/null +++ b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/build.sh @@ -0,0 +1,359 @@ +#!/usr/bin/env bash +# ============================================================================= +# build.sh — local equivalent of the Jenkinsfile pipeline +# +# Stages: +# 1. Clean workspace + podman prune +# 2. Checkout opensearch-build (tag 3.5.0) and build-scripts (opensearch-3.5.0-ais) +# 3. Build the build-env image +# 4. Apply patch, run build.sh + assemble.sh inside container, copy dist tar +# 5. Build the final OpenSearch container image +# 6. Run Trivy security scans (JSON + table), save reports to ARTIFACTS_DIR +# 7. Push image to ICR (requires IBM_CLOUD_APIKEY env var) +# +# Artifacts saved to ARTIFACTS_DIR (default: /var/jenkins/artifacts/opensearch): +# - opensearch-3.5.0-linux-ppc64le.tar.gz (assembled dist) +# - trivy-report/trivy-results.json +# - trivy-report/trivy-results.txt +# +# Usage: +# export IBM_CLOUD_APIKEY="" +# ./build.sh [--skip-push] [--skip-scan] [--resume-from ] +# +# Flags: +# --skip-push Skip the ICR push stage +# --skip-scan Skip the Trivy scan stage +# --resume-from Resume from stage N; useful after a failed run +# where earlier stages already succeeded. +# Stages: 1=clean 2=checkout 3=build-env 4=artifacts +# 5=container-image 6=scan 7=push +# Example: --resume-from 5 +# ============================================================================= + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration (matches Jenkinsfile environment block) +# --------------------------------------------------------------------------- +OPENSEARCH_VERSION="3.5.0" +ARCHITECTURE="ppc64le" +PLATFORM="linux" + +# Where to persist final artifacts on the host +ARTIFACTS_DIR="${ARTIFACTS_DIR:-/var/jenkins/artifacts/opensearch}" + +# Repositories +OPENSEARCH_BUILD_REPO="https://github.com/opensearch-project/opensearch-build.git" +OPENSEARCH_BUILD_BRANCH="3.5.0" # checked out as a tag + +BUILD_SCRIPTS_REPO="https://github.com/irapandey/build-scripts.git" +BUILD_SCRIPTS_BRANCH="opensearch-3.5.0-ais" + +# Working directories (everything lives under WORK_DIR; easy to wipe) +WORK_DIR="${WORK_DIR:-$(pwd)/workspace}" +BUILD_DIR="${WORK_DIR}/opensearch-build" +SCRIPTS_DIR="${WORK_DIR}/build-scripts" + +# Paths inside build-scripts repo +PATCH_FILE="o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/ppc64le-3.5.0-ai-services.patch" +DOCKERFILE_DIR="o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7" + +# Build-env image name +BUILD_ENV_IMAGE="opensearch-build-env:3.5.0-ppc64le" +BUILD_ENV_DOCKERFILE="${DOCKERFILE_DIR}/Dockerfile.build-env" + +# Final image +ICR_REGISTRY="icr.io" +IMAGE_NAME="icr.io/ai-services-private/opensearch:${OPENSEARCH_VERSION}-ppc64le-extended" + +# Trivy settings +TRIVY_IMAGE="ghcr.io/aquasecurity/trivy:0.72.0" +TRIVY_CACHE="${TRIVY_CACHE:-/var/jenkins/trivy-cache}" + +# --------------------------------------------------------------------------- +# Flag defaults +# --------------------------------------------------------------------------- +SKIP_PUSH=false +SKIP_SCAN=false +RESUME_FROM=1 + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-push) SKIP_PUSH=true; shift ;; + --skip-scan) SKIP_SCAN=true; shift ;; + --resume-from) RESUME_FROM="$2"; shift 2 ;; + *) + echo "Unknown option: $1" >&2 + echo "Usage: $0 [--skip-push] [--skip-scan] [--resume-from <1|2|3|4|5|6|7>]" >&2 + exit 1 + ;; + esac +done + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +log() { echo -e "\n\033[1;36m>>> [$(date '+%H:%M:%S')] $*\033[0m"; } +ok() { echo -e "\033[1;32m ✔ $*\033[0m"; } +warn() { echo -e "\033[1;33m ⚠ $*\033[0m"; } +die() { echo -e "\033[1;31m ✘ $*\033[0m" >&2; exit 1; } + +# Ordered stage list — used to resolve --resume-from comparisons. +STAGE_ORDER=(1 2 3 4 5 6 7) + +stage_index() { + local s="$1" i=0 + for stage in "${STAGE_ORDER[@]}"; do + if [[ "$stage" == "$s" ]]; then echo "$i"; return; fi + (( i++ )) + done + echo "-1" +} + +# Validate RESUME_FROM after helpers are defined (checked at first call below). +_RESUME_IDX="" +_validate_resume_from() { + if [[ -z "${_RESUME_IDX}" ]]; then + _RESUME_IDX=$(stage_index "${RESUME_FROM}") + [[ "${_RESUME_IDX}" -ge 0 ]] || \ + die "Invalid --resume-from value '${RESUME_FROM}'. Valid: 1 2 3 4 5 6 7" + fi +} + +stage_should_run() { + # Returns 0 (true) when this stage's index >= the resume-from stage's index + _validate_resume_from + local this_idx + this_idx=$(stage_index "$1") + [[ "${this_idx}" -ge "${_RESUME_IDX}" ]] +} + +# --------------------------------------------------------------------------- +# Stage 1 — Clean workspace + reclaim podman space +# --------------------------------------------------------------------------- +if stage_should_run 1; then + log "STAGE 1 — Clean workspace" + + log " Reclaiming Podman space..." + podman system prune --force --volumes + podman image prune --force --all --filter "label!=keep" || true + + echo "=== Disk after prune ===" + df -h / + + log " Removing previous workspace: ${WORK_DIR}" + rm -rf "${WORK_DIR}" + mkdir -p "${WORK_DIR}" + + ok "Stage 1 complete" +fi + +# --------------------------------------------------------------------------- +# Stage 2 — Checkout sources +# --------------------------------------------------------------------------- +if stage_should_run 2; then + log "STAGE 2 — Checkout sources" + + log " Cloning opensearch-build @ tag ${OPENSEARCH_BUILD_BRANCH}..." + mkdir -p "${BUILD_DIR}" + git clone --depth=1 \ + --branch "${OPENSEARCH_BUILD_BRANCH}" \ + "${OPENSEARCH_BUILD_REPO}" \ + "${BUILD_DIR}" + + log " Cloning build-scripts @ branch ${BUILD_SCRIPTS_BRANCH}..." + mkdir -p "${SCRIPTS_DIR}" + git clone --depth=1 \ + --branch "${BUILD_SCRIPTS_BRANCH}" \ + "${BUILD_SCRIPTS_REPO}" \ + "${SCRIPTS_DIR}" + + ok "Stage 2 complete" +fi + +# --------------------------------------------------------------------------- +# Stage 3 — Build the build-env container image +# --------------------------------------------------------------------------- +if stage_should_run 3; then + log "STAGE 3 — Build build-env image" + + echo "=== Disk space ===" + df -h + echo "=== Memory ===" + free -h + echo "=== Podman disk usage ===" + podman system df || true + + podman build \ + -t "${BUILD_ENV_IMAGE}" \ + -f "${SCRIPTS_DIR}/${BUILD_ENV_DOCKERFILE}" \ + "${BUILD_DIR}" + + ok "Stage 3 complete — image: ${BUILD_ENV_IMAGE}" +fi + +# --------------------------------------------------------------------------- +# Stage 4 — Apply patch, build artifacts, assemble dist, copy to host +# --------------------------------------------------------------------------- +if stage_should_run 4; then + log "STAGE 4 — Build artifacts and assemble distribution (full manifest)" + + log " Applying patch..." + git config --global user.name "OpenSearch Build" + git config --global user.email "opensearch-build@example.com" + + # Apply all shell/python changes to the opensearch-build clone. + git -C "${BUILD_DIR}" apply \ + --whitespace=nowarn \ + "${SCRIPTS_DIR}/${PATCH_FILE}" + + log " Running OpenSearch build + assemble inside container..." + podman run --rm \ + --cpus=4 \ + --ulimit nproc=65536:65536 \ + -v "${BUILD_DIR}:/opensearch-build:z" \ + -w /opensearch-build \ + --entrypoint '' \ + "${BUILD_ENV_IMAGE}" \ + bash -c 'set -euo pipefail && \ + ./build.sh \ + manifests/3.5.0/opensearch-3.5.0.yml \ + --platform '"${PLATFORM}"' \ + --architecture '"${ARCHITECTURE}"' && \ + ./assemble.sh tar/builds/opensearch/manifest.yml' + + log " Copying assembled tar to host..." + mkdir -p "${ARTIFACTS_DIR}" + cp "${BUILD_DIR}/tar/dist/opensearch/opensearch-${OPENSEARCH_VERSION}-linux-${ARCHITECTURE}.tar.gz" \ + "${ARTIFACTS_DIR}/opensearch-${OPENSEARCH_VERSION}-linux-${ARCHITECTURE}.tar.gz" + + ok "Stage 4 complete — tar at ${ARTIFACTS_DIR}/opensearch-${OPENSEARCH_VERSION}-linux-${ARCHITECTURE}.tar.gz" +fi + +# --------------------------------------------------------------------------- +# Stage 5 — Build final container image +# --------------------------------------------------------------------------- +if stage_should_run 5; then + log "STAGE 5 — Build container image" + + DOCKER_CONTEXT="${WORK_DIR}/docker-context" + rm -rf "${DOCKER_CONTEXT}" + mkdir -p "${DOCKER_CONTEXT}" + + # Dockerfile and supporting files from build-scripts + cp -r "${SCRIPTS_DIR}/${DOCKERFILE_DIR}/"* "${DOCKER_CONTEXT}/" + + # Entrypoint scripts and configs from opensearch-build repo + cp "${BUILD_DIR}/docker/release/config/opensearch/opensearch-docker-entrypoint-"*.x.sh "${DOCKER_CONTEXT}/" + cp "${BUILD_DIR}/docker/release/config/opensearch/log4j2.properties" "${DOCKER_CONTEXT}/" + cp "${BUILD_DIR}/scripts/opensearch-onetime-setup.sh" "${DOCKER_CONTEXT}/" + cp "${BUILD_DIR}/config/opensearch.yml" "${DOCKER_CONTEXT}/" + + # Assembled tarball — rename to what the Dockerfile expects + cp "${BUILD_DIR}/tar/dist/opensearch/opensearch-${OPENSEARCH_VERSION}-linux-${ARCHITECTURE}.tar.gz" \ + "${DOCKER_CONTEXT}/opensearch-ppc64le.tgz" + + BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + podman build \ + --build-arg "VERSION=${OPENSEARCH_VERSION}" \ + --build-arg "BUILD_DATE=${BUILD_DATE}" \ + --build-arg "UID=1000" \ + --build-arg "GID=1000" \ + -f "${DOCKER_CONTEXT}/Dockerfile.ais" \ + -t "${IMAGE_NAME}" \ + "${DOCKER_CONTEXT}" + + ok "Stage 5 complete — image: ${IMAGE_NAME}" +fi + +# --------------------------------------------------------------------------- +# Stage 6 — Trivy security scan +# --------------------------------------------------------------------------- +if stage_should_run 6 && [[ "${SKIP_SCAN}" == false ]]; then + log "STAGE 6 — Trivy security scan" + + REPORT_DIR="${ARTIFACTS_DIR}/trivy-report" + IMAGE_ARCHIVE="${REPORT_DIR}/image.tar" + mkdir -p "${REPORT_DIR}" "${TRIVY_CACHE}" + + log " Exporting ${IMAGE_NAME} to Docker archive..." + podman save --format docker-archive -o "${IMAGE_ARCHIVE}" "${IMAGE_NAME}" + + log " Running Trivy scan — JSON report..." + podman run --rm \ + -v "${REPORT_DIR}:/report:z" \ + -v "${TRIVY_CACHE}:/root/.cache/trivy:z" \ + "${TRIVY_IMAGE}" image \ + --input /report/image.tar \ + --format json \ + --output /report/trivy-results.json \ + --severity CRITICAL,HIGH \ + --ignore-unfixed \ + --scanners vuln + + log " Running Trivy scan — table report..." + podman run --rm \ + -v "${REPORT_DIR}:/report:z" \ + -v "${TRIVY_CACHE}:/root/.cache/trivy:z" \ + "${TRIVY_IMAGE}" image \ + --input /report/image.tar \ + --format table \ + --output /report/trivy-results.txt \ + --severity CRITICAL,HIGH \ + --ignore-unfixed \ + --scanners vuln + + echo "" + echo "=== Trivy Summary (CRITICAL/HIGH counts) ===" + grep -E '"Severity"' "${REPORT_DIR}/trivy-results.json" \ + | sort | uniq -c | sort -rn || true + + # Remove the archive to reclaim disk space + rm -f "${IMAGE_ARCHIVE}" + + ok "Stage 6 complete — reports at ${REPORT_DIR}/" +elif [[ "${SKIP_SCAN}" == true ]]; then + warn "Stage 6 (Trivy scan) skipped via --skip-scan" +fi + +# # --------------------------------------------------------------------------- +# # Stage 7 — Push image to ICR +# # --------------------------------------------------------------------------- +# if stage_should_run 7 && [[ "${SKIP_PUSH}" == false ]]; then +# log "STAGE 7 — Push image to ICR" + +# if [[ -z "${IBM_CLOUD_APIKEY:-}" ]]; then +# die "IBM_CLOUD_APIKEY is not set. Export it before running, or use --skip-push." +# fi + +# log " Logging into ${ICR_REGISTRY}..." +# set +x +# echo "${IBM_CLOUD_APIKEY}" | podman login \ +# -u iamapikey \ +# --password-stdin \ +# "${ICR_REGISTRY}" +# set -x + +# log " Pushing ${IMAGE_NAME}..." +# podman push "${IMAGE_NAME}" + +# podman logout "${ICR_REGISTRY}" + +# ok "Stage 7 complete — image pushed: ${IMAGE_NAME}" +# elif [[ "${SKIP_PUSH}" == true ]]; then +# warn "Stage 7 (ICR push) skipped via --skip-push" +# fi + +# --------------------------------------------------------------------------- +# Done +# --------------------------------------------------------------------------- +log "All stages complete." +echo "" +echo " Dist tar : ${ARTIFACTS_DIR}/opensearch-${OPENSEARCH_VERSION}-linux-${ARCHITECTURE}.tar.gz" +echo " Trivy JSON : ${ARTIFACTS_DIR}/trivy-report/trivy-results.json" +echo " Trivy TXT : ${ARTIFACTS_DIR}/trivy-report/trivy-results.txt" +echo " Image : ${IMAGE_NAME}" diff --git a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/opensearch-build.patch b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/opensearch-build.patch deleted file mode 100644 index 113c5f2736..0000000000 --- a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/opensearch-build.patch +++ /dev/null @@ -1,113 +0,0 @@ -diff --git a/scripts/components/OpenSearch/build.sh b/scripts/components/OpenSearch/build.sh -index 0797262e..6be344a8 100755 ---- a/scripts/components/OpenSearch/build.sh -+++ b/scripts/components/OpenSearch/build.sh -@@ -75,11 +75,11 @@ mkdir -p $OUTPUT/maven/org/opensearch - - # Build project and publish to maven local. - echo "Building and publishing OpenSearch project to Maven Local" --./gradlew publishToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain publishToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - - # Publish to existing test repo, using this to stage release versions of the artifacts that can be released from the same build. - echo "Publishing OpenSearch to Test Repository" --./gradlew publishNebulaPublicationToTestRepository -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain publishNebulaPublicationToTestRepository -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - - # Copy maven publications to be promoted - echo "Copying Maven publications to $OUTPUT/maven/org" -@@ -164,7 +164,7 @@ esac - - echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" - --./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - - # Copy artifact to dist folder in bundle build output - echo "Copying artifact to ${OUTPUT}/dist" -@@ -176,7 +176,7 @@ cp -v distribution/$TYPE/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME "${OUT - echo "Building core plugins..." - mkdir -p "${OUTPUT}/core-plugins" - cd plugins --../gradlew assemble -Dbuild.snapshot="$SNAPSHOT" -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+../gradlew --console=plain assemble -Dbuild.snapshot="$SNAPSHOT" -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - cd .. - for plugin in plugins/*; do - PLUGIN_NAME=$(basename "$plugin") -diff --git a/scripts/components/notifications-core/build.sh b/scripts/components/notifications-core/build.sh -index b162b4e0..3df3a026 100644 ---- a/scripts/components/notifications-core/build.sh -+++ b/scripts/components/notifications-core/build.sh -@@ -68,13 +68,13 @@ fi - [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT - [ -z "$OUTPUT" ] && OUTPUT=artifacts - --./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 --./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - - mkdir -p ./$OUTPUT/plugins - notifCoreZipPath=$(ls core/build/distributions/ | grep .zip) - cp -v core/build/distributions/$notifCoreZipPath ./$OUTPUT/plugins - --./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - mkdir -p $OUTPUT/maven/org/opensearch/plugin - cp -r ./build/local-staging-repo/org/opensearch/plugin/opensearch-notifications-core $OUTPUT/maven/org/opensearch/plugin/ -diff --git a/scripts/components/notifications/build.sh b/scripts/components/notifications/build.sh -index bdadef1b..8f961b29 100644 ---- a/scripts/components/notifications/build.sh -+++ b/scripts/components/notifications/build.sh -@@ -68,14 +68,14 @@ fi - [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT - [ -z "$OUTPUT" ] && OUTPUT=artifacts - --./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 --./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - - mkdir -p ./$OUTPUT/plugins - - notifCoreZipPath=$(ls notifications/build/distributions/ | grep .zip) - cp -v notifications/build/distributions/$notifCoreZipPath ./$OUTPUT/plugins - --./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - mkdir -p $OUTPUT/maven/org/opensearch/plugin - cp -r ./build/local-staging-repo/org/opensearch/plugin/notifications $OUTPUT/maven/org/opensearch/plugin/ -diff --git a/scripts/default/opensearch/build.sh b/scripts/default/opensearch/build.sh -index 518c722a..3b8b1e61 100755 ---- a/scripts/default/opensearch/build.sh -+++ b/scripts/default/opensearch/build.sh -@@ -70,7 +70,7 @@ fi - - mkdir -p $OUTPUT - --./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain --console=plain --console=plain assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 - - zipPath=$(find . -path \*build/distributions/*.zip) - distributions="$(dirname "${zipPath}")" -@@ -80,7 +80,7 @@ mkdir -p $OUTPUT/plugins - cp ${distributions}/*.zip ./$OUTPUT/plugins - - # Publish plugin zips to maven --./gradlew publishPluginZipPublicationToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 --./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain --console=plain --console=plain publishPluginZipPublicationToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-140-3 -+./gradlew --console=plain --console=plain --console=plain publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Pcrypto.standard=FIPS-139-3 - mkdir -p $OUTPUT/maven/org/opensearch - cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch -diff --git a/src/build_workflow/build_args.py b/src/build_workflow/build_args.py -index d67c27a7..271962ab 100644 ---- a/src/build_workflow/build_args.py -+++ b/src/build_workflow/build_args.py -@@ -16,6 +16,7 @@ class BuildArgs: - SUPPORTED_ARCHITECTURES = [ - "x64", - "arm64", -+ "ppc64le", - ] - SUPPORTED_DISTRIBUTIONS = ["tar", "zip", "rpm", "deb"] \ No newline at end of file diff --git a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/opensearch-docker-build.patch b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/opensearch-docker-build.patch deleted file mode 100644 index fa582df922..0000000000 --- a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/opensearch-docker-build.patch +++ /dev/null @@ -1,21 +0,0 @@ -diff --git a/docker/release/config/opensearch/opensearch-docker-entrypoint-3.x.sh b/docker/release/config/opensearch/opensearch-docker-entrypoint-3.x.sh -index d9986638..34a5f93d 100755 ---- a/docker/release/config/opensearch/opensearch-docker-entrypoint-3.x.sh -+++ b/docker/release/config/opensearch/opensearch-docker-entrypoint-3.x.sh -@@ -12,6 +12,12 @@ - export OPENSEARCH_HOME=/usr/share/opensearch - export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config - -+# Ensure FIPS mode is disabled for BouncyCastle to allow standard passwords -+# This must be set before any Java operations that use the keystore -+if [[ ! "$OPENSEARCH_JAVA_OPTS" =~ "org.bouncycastle.fips.approved_only" ]]; then -+ export OPENSEARCH_JAVA_OPTS="-Dorg.bouncycastle.fips.approved_only=false $OPENSEARCH_JAVA_OPTS" -+fi -+ - # The virtual file /proc/self/cgroup should list the current cgroup - # membership. For each hierarchy, you can follow the cgroup path from - # this file to the cgroup filesystem (usually /sys/fs/cgroup/) and -@@ -114,3 +120,4 @@ else - # Otherwise, just exec the command. - exec "$@" - fi \ No newline at end of file diff --git a/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/ppc64le-3.5.0-ai-services.patch b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/ppc64le-3.5.0-ai-services.patch new file mode 100644 index 0000000000..4134ed5610 --- /dev/null +++ b/o/opensearch-project-opensearch-build/Dockerfiles/3.5.0_ubi9.7/ppc64le-3.5.0-ai-services.patch @@ -0,0 +1,1379 @@ +From ce984e595b00cab581c48c57f2227eb3f56df4ff Mon Sep 17 00:00:00 2001 +From: irapandey +Date: Tue, 15 Sep 2026 17:19:27 +0530 +Subject: [PATCH] ppc64le-ccr + +Signed-off-by: irapandey +--- + common-utils.patch | 20 +++ + cross-cluster-replication.patch | 73 ++++++++ + knn.patch | 21 +++ + manifests/3.5.0/opensearch-3.5.0.yml | 157 ------------------ + opensearch.patch | 50 ++++++ + .../components/OpenSearch-DataFusion/build.sh | 24 ++- + scripts/components/OpenSearch/build.sh | 24 ++- + scripts/components/OpenSearch/integtest.sh | 11 +- + scripts/components/alerting/integtest.sh | 11 +- + scripts/components/k-NN/build.sh | 127 ++++++++++++++ + scripts/components/k-NN/integtest.sh | 13 +- + .../components/notifications-core/build.sh | 16 +- + scripts/components/notifications/build.sh | 16 +- + scripts/default/bwctest.sh | 11 +- + scripts/default/integtest.sh | 11 +- + scripts/default/opensearch/build.sh | 16 +- + security.patch | 69 ++++++++ + src/build_workflow/build_args.py | 1 + + src/build_workflow/builder_from_source.py | 122 +++++++++++++- + .../ci_check_gradle_dependencies.py | 2 +- + src/ci_workflow/ci_check_gradle_properties.py | 2 +- + .../ci_check_gradle_publish_to_maven_local.py | 2 +- + .../component_opensearch.py | 2 +- + .../test_ci_check_gradle_dependencies.py | 10 +- + ...ci_check_gradle_dependencies_opensearch.py | 8 +- + .../test_ci_check_gradle_properties.py | 6 +- + ..._ci_check_gradle_publish_to_maven_local.py | 6 +- + .../test_component_opensearch.py | 6 +- + .../wss-unified-agent.config | 2 +- + 29 files changed, 626 insertions(+), 213 deletions(-) + create mode 100644 common-utils.patch + create mode 100644 cross-cluster-replication.patch + create mode 100644 knn.patch + create mode 100644 opensearch.patch + create mode 100755 scripts/components/k-NN/build.sh + create mode 100644 security.patch + +diff --git a/common-utils.patch b/common-utils.patch +new file mode 100644 +index 00000000..45158fab +--- /dev/null ++++ b/common-utils.patch +@@ -0,0 +1,20 @@ ++diff --git a/scripts/build.sh b/scripts/build.sh ++index e0495d4..71b710b 100755 ++--- a/scripts/build.sh +++++ b/scripts/build.sh ++@@ -64,8 +64,8 @@ fi ++ [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT ++ [ -z "$OUTPUT" ] && OUTPUT=artifacts ++ ++-./gradlew build -x test -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++-./gradlew publishShadowPublicationToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++-./gradlew publishShadowPublicationToStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +++./gradlew --console=plain build -x test -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +++./gradlew --console=plain publishShadowPublicationToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +++./gradlew --console=plain publishShadowPublicationToStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++ mkdir -p $OUTPUT/maven/org/opensearch ++ cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch ++-- ++2.51.2 ++ ++ +diff --git a/cross-cluster-replication.patch b/cross-cluster-replication.patch +new file mode 100644 +index 00000000..c72b7a3f +--- /dev/null ++++ b/cross-cluster-replication.patch +@@ -0,0 +1,73 @@ ++diff --git a/build.gradle b/build.gradle ++index b1e1ac0..34c406b 100644 ++--- a/build.gradle +++++ b/build.gradle ++@@ -188,15 +188,12 @@ compileTestKotlin { ++ } ++ } ++ ++-ext { ++- licenseFile = rootProject.file('LICENSE') ++- noticeFile = rootProject.file('NOTICE') ++-} ++- ++ opensearchplugin { ++ name = project.name ++ description = "OpenSearch Cross Cluster Replication Plugin" ++ classname = "org.opensearch.replication.ReplicationPlugin" +++ licenseFile rootProject.file('LICENSE') +++ noticeFile rootProject.file('NOTICE') ++ } ++ ++ java { ++diff --git a/gradle.properties b/gradle.properties ++new file mode 100644 ++index 0000000..2b62832 ++--- /dev/null +++++ b/gradle.properties ++@@ -0,0 +1,6 @@ +++# ppc64le: disable native-platform library (no ppc64le binaries available). +++# Falls back to pure Java implementations. +++org.gradle.native=false +++ +++# Use plain console output (avoids native terminal formatting requirements). +++org.gradle.console=plain ++diff --git a/init.d/ppc64le-kotlin-fix.gradle b/init.d/ppc64le-kotlin-fix.gradle ++new file mode 100644 ++index 0000000..121d0db ++--- /dev/null +++++ b/init.d/ppc64le-kotlin-fix.gradle ++@@ -0,0 +1,30 @@ +++/* +++ * SPDX-License-Identifier: Apache-2.0 +++ * +++ * ppc64le build fix for Kotlin 2.x Gradle plugin. +++ * +++ * Problem: +++ * Kotlin 2.x introduces NativeVersionChecker, which triggers a static +++ * class initialiser in NativeCompilerDownloader at Gradle configuration +++ * time. That initialiser calls HostManager.hostArch(), which reads +++ * System.getProperty("os.arch") and throws: +++ * +++ * TargetSupportException: Unknown hardware platform: ppc64le +++ * +++ * This aborts build-script evaluation before the opensearchplugin block +++ * is even reached, causing a cascade of misleading secondary errors. +++ * +++ * Fix: +++ * Spoof os.arch to "x86_64" in a Gradle init script so that the property +++ * is already set before any plugin classloading occurs. The override has +++ * no effect on the JVM compilation output — all Kotlin/Java targets remain +++ * JVM bytecode regardless of the reported host architecture. +++ * +++ * Usage: +++ * Copy (or symlink) this file to ~/.gradle/init.d/ on any ppc64le build host. +++ * It is a no-op on x86_64/aarch64 hosts because those values are already +++ * accepted by HostManager.hostArch(). +++ */ +++if (System.getProperty("os.arch") == "ppc64le") { +++ System.setProperty("os.arch", "x86_64") +++} ++-- ++2.51.2 ++ +diff --git a/knn.patch b/knn.patch +new file mode 100644 +index 00000000..399a7e37 +--- /dev/null ++++ b/knn.patch +@@ -0,0 +1,21 @@ ++diff --git a/build.gradle b/build.gradle ++index 54740d0a..4a87e896 100644 ++--- a/build.gradle +++++ b/build.gradle ++@@ -219,6 +219,15 @@ def getBreakerSetting() { ++ ++ allprojects { ++ +++ configurations.all { +++ resolutionStrategy { +++ force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" +++ force "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" +++ force "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" +++ force "org.apache.httpcomponents.core5:httpcore5-reactive:${versions.httpcore5}" +++ } +++ } +++ ++ group = 'org.opensearch' ++ version = opensearch_version.tokenize('-')[0] + '.0' ++ if (version_qualifier) { ++ +diff --git a/manifests/3.5.0/opensearch-3.5.0.yml b/manifests/3.5.0/opensearch-3.5.0.yml +index caa45687..76e2cafa 100644 +--- a/manifests/3.5.0/opensearch-3.5.0.yml ++++ b/manifests/3.5.0/opensearch-3.5.0.yml +@@ -29,12 +29,6 @@ components: + platforms: + - linux + - windows +- - name: opensearch-learning-to-rank-base +- repository: https://github.com/opensearch-project/opensearch-learning-to-rank-base.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows + - name: opensearch-remote-metadata-sdk + repository: https://github.com/opensearch-project/opensearch-remote-metadata-sdk.git + ref: tags/3.5.0.0 +@@ -61,22 +55,6 @@ components: + - windows + depends_on: + - common-utils +- - name: geospatial +- repository: https://github.com/opensearch-project/geospatial.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - job-scheduler +- - name: cross-cluster-replication +- repository: https://github.com/opensearch-project/cross-cluster-replication.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: tags/3.5.0.0 +@@ -96,24 +74,6 @@ components: + depends_on: + - ml-commons + - k-NN +- - name: notifications-core +- repository: https://github.com/opensearch-project/notifications.git +- ref: tags/3.5.0.0 +- working_directory: notifications +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils +- - name: notifications +- repository: https://github.com/opensearch-project/notifications.git +- ref: tags/3.5.0.0 +- working_directory: notifications +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils + - name: opensearch-observability + repository: https://github.com/opensearch-project/observability.git + ref: tags/3.5.0.0 +@@ -122,59 +82,6 @@ components: + - windows + depends_on: + - common-utils +- - name: opensearch-reports +- repository: https://github.com/opensearch-project/reporting.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils +- - job-scheduler +- - name: sql +- repository: https://github.com/opensearch-project/sql.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - ml-commons +- - geospatial +- - name: asynchronous-search +- repository: https://github.com/opensearch-project/asynchronous-search.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils +- - name: anomaly-detection +- repository: https://github.com/opensearch-project/anomaly-detection.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils +- - job-scheduler +- - name: alerting +- repository: https://github.com/opensearch-project/alerting.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils +- - name: security-analytics +- repository: https://github.com/opensearch-project/security-analytics.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils +- - alerting +- - job-scheduler + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: tags/3.5.0.0 +@@ -184,67 +91,3 @@ components: + depends_on: + - common-utils + - job-scheduler +- - name: performance-analyzer +- repository: https://github.com/opensearch-project/performance-analyzer.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - name: custom-codecs +- repository: https://github.com/opensearch-project/custom-codecs.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- - name: flow-framework +- repository: https://github.com/opensearch-project/flow-framework.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - common-utils +- - opensearch-remote-metadata-sdk +- - ml-commons +- - k-NN +- - neural-search +- - name: skills +- repository: https://github.com/opensearch-project/skills.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - job-scheduler +- - anomaly-detection +- - sql +- - ml-commons +- - name: query-insights +- repository: https://github.com/opensearch-project/query-insights.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- - name: opensearch-system-templates +- repository: https://github.com/opensearch-project/opensearch-system-templates.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- - name: user-behavior-insights +- repository: https://github.com/opensearch-project/user-behavior-insights.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- - name: search-relevance +- repository: https://github.com/opensearch-project/search-relevance.git +- ref: tags/3.5.0.0 +- platforms: +- - linux +- - windows +- depends_on: +- - job-scheduler +- - neural-search +- - k-NN +- - ml-commons +- - user-behavior-insights +diff --git a/opensearch.patch b/opensearch.patch +new file mode 100644 +index 00000000..1502c143 +--- /dev/null ++++ b/opensearch.patch +@@ -0,0 +1,50 @@ ++diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java ++index b75bdcff..dce9ca76 100644 ++--- a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java +++++ b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java ++@@ -116,7 +116,7 @@ public abstract class DockerSupportService implements BuildService /dev/null 2>&1) || (rpm -q opensearch > /dev/null 2>&1); then + echo "Run systemd integTest for OpenSearch core engine" +- ./gradlew qa:systemd-test:integTest --tests org.opensearch.systemdinteg.SystemdIntegTests --console=plain ++ ./gradlew --console=plain qa:systemd-test:integTest --tests org.opensearch.systemdinteg.SystemdIntegTests --console=plain + else + echo "No deb or rpm installed detected, skip test" + fi +diff --git a/scripts/components/alerting/integtest.sh b/scripts/components/alerting/integtest.sh +index b12b5e5d..4939ed02 100755 +--- a/scripts/components/alerting/integtest.sh ++++ b/scripts/components/alerting/integtest.sh +@@ -102,4 +102,13 @@ fi + USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` + PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` + +-./gradlew integTest -Dopensearch.version=$OPENSEARCH_VERSION -Dbuild.snapshot=$SNAPSHOT -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Dsecurity=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain ++# On ppc64le, the kernel thread limit is tighter than on x86_64/arm64. ++# Gradle's default worker count (= CPU cores) combined with JVM GC threads ++# exhausts pthread resources, causing "pthread_create failed (EAGAIN)" and ++# preventing subprocesses such as javadoc from starting. Cap workers to 4 ++# to stay well within the limit on constrained ppc64le CI runners. ++if [ "$(uname -m)" = "ppc64le" ]; then ++ export GRADLE_OPTS="${GRADLE_OPTS} -Dorg.gradle.workers.max=4" ++fi ++ ++./gradlew --console=plain integTest -Dopensearch.version=$OPENSEARCH_VERSION -Dbuild.snapshot=$SNAPSHOT -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Dsecurity=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain +diff --git a/scripts/components/k-NN/build.sh b/scripts/components/k-NN/build.sh +new file mode 100755 +index 00000000..8dff129d +--- /dev/null ++++ b/scripts/components/k-NN/build.sh +@@ -0,0 +1,127 @@ ++#!/bin/bash ++ ++# Copyright OpenSearch Contributors ++# SPDX-License-Identifier: Apache-2.0 ++# ++# The OpenSearch Contributors require contributions made to ++# this file be licensed under the Apache-2.0 license or a ++# compatible open source license. ++ ++# Wrapper build script for k-NN on ppc64le. ++# ++# The k-NN plugin compiles JNI native libraries (Faiss, NMSLIB, SIMD) via CMake. ++# The upstream build script notes "Linux versions already have OpenBlas in the runner" ++# — true for x64/arm64 runners, but ppc64le runners do not have these packages ++# pre-installed. We install the required native build dependencies here before ++# delegating to the upstream build script that lives in the cloned k-NN repository. ++# ++# Required packages: ++# cmake - build system used by the JNI CMakeLists.txt ++# openblas-devel - BLAS/LAPACK implementation; Faiss requires find_package(BLAS/LAPACK REQUIRED); ++# on RHEL 9 ppc64le this package also provides the LAPACK headers so lapack-devel ++# is not separately available and not needed ++# gcc-gfortran - Fortran compiler; Faiss calls enable_language(Fortran) ++# libgomp - OpenMP runtime; Faiss links OpenMP::OpenMP_CXX ++# ++# Note: openblas-static and lapack-devel are NOT available in the RHEL 9 ppc64le repositories ++# and are therefore not listed here. ++# ++# ppc64le compiler-flag fix: ++# nmslib's CMakeLists.txt sets -march=native which is an x86/arm GCC flag. On ppc64le ++# GCC does not accept -march at all. ++# ++# We cannot patch the source file directly: the k-NN repo's cmake/init-nmslib.cmake ++# applies a git patch to nmslib via "git apply" after submodule init. If the working ++# tree has been modified (even by sed), git apply fails with: ++# error: similarity_search/CMakeLists.txt: does not match index ++# ++# We also cannot use -mcpu=native: GCC resolves "native" by querying the kernel, and ++# on a Power11 host running GCC 11 (RHEL 9) the kernel reports "power11" which GCC 11 ++# does not know yet (its highest is "power10"). The result is: ++# unsupported cpu name returned from kernel for '-mcpu=native': power11 ++# ++# Solution: install a thin compiler wrapper that replaces -march=native with ++# -mcpu=power10 (the newest micro-arch GCC 11 knows; power10 code runs correctly on ++# power11 due to ISA backward compatibility). The wrapper is a small shell script ++# created in a per-build temp dir; it does not touch any source file and is completely ++# transparent to git. cmake picks up $CXX/$CC from the environment and records the ++# wrapper path in CMakeCache.txt, so every cmake invocation uses it. ++ ++set -ex ++ ++[ -z "$ARCHITECTURE" ] && ARCHITECTURE=$(uname -m) ++ ++if [ "$ARCHITECTURE" = "ppc64le" ]; then ++ PKG_MANAGER="" ++ if command -v dnf &>/dev/null; then ++ PKG_MANAGER="dnf" ++ elif command -v yum &>/dev/null; then ++ PKG_MANAGER="yum" ++ fi ++ ++ if [ -n "$PKG_MANAGER" ]; then ++ MISSING_PKGS=() ++ command -v cmake &>/dev/null || MISSING_PKGS+=(cmake) ++ rpm -q openblas-devel &>/dev/null || MISSING_PKGS+=(openblas-devel) ++ rpm -q gcc-gfortran &>/dev/null || MISSING_PKGS+=(gcc-gfortran) ++ rpm -q libgomp &>/dev/null || MISSING_PKGS+=(libgomp) ++ ++ if [ "${#MISSING_PKGS[@]}" -gt 0 ]; then ++ echo "Installing missing native build dependencies on ppc64le: ${MISSING_PKGS[*]}" ++ $PKG_MANAGER install -y "${MISSING_PKGS[@]}" ++ fi ++ fi ++ ++ # On ppc64le the kernel thread limit is tighter than on x86_64/arm64. ++ # Cap Gradle worker count to stay within pthread resource limits. ++ export GRADLE_OPTS="${GRADLE_OPTS} -Dorg.gradle.workers.max=4" ++ ++ # Install a compiler wrapper that rewrites -march=native -> -mcpu=native. ++ # This avoids touching any source file (which would break the upstream git-apply ++ # patch in cmake/init-nmslib.cmake). The wrapper delegates to the real c++ ++ # compiler after substituting the flag. ++ _CXX_WRAPPER_DIR=$(mktemp -d) ++ _REAL_CXX=$(command -v c++) ++ cat > "$_CXX_WRAPPER_DIR/c++" <<'WRAPPER' ++#!/bin/bash ++# ppc64le shim: replace -march=native with -mcpu=power10. ++# -march=native is x86-only; -mcpu=native would work but GCC 11 on RHEL 9 does not ++# recognise 'power11' returned by the kernel on Power11 hosts. power10 is the newest ++# micro-arch GCC 11 knows and runs correctly on power11 (ISA backward compatibility). ++REAL_CXX="@REAL_CXX@" ++args=() ++for arg in "$@"; do ++ args+=("${arg/-march=native/-mcpu=power10}") ++done ++exec "$REAL_CXX" "${args[@]}" ++WRAPPER ++ # Substitute the real compiler path into the wrapper. ++ sed -i "s|@REAL_CXX@|${_REAL_CXX}|g" "$_CXX_WRAPPER_DIR/c++" ++ chmod +x "$_CXX_WRAPPER_DIR/c++" ++ ++ # Also wrap 'cc' / 'gcc' in case cmake probes the C compiler for the same flag. ++ _REAL_CC=$(command -v cc) ++ cat > "$_CXX_WRAPPER_DIR/cc" <<'WRAPPER' ++#!/bin/bash ++# ppc64le shim: same flag replacement as the c++ wrapper above. ++REAL_CC="@REAL_CC@" ++args=() ++for arg in "$@"; do ++ args+=("${arg/-march=native/-mcpu=power10}") ++done ++exec "$REAL_CC" "${args[@]}" ++WRAPPER ++ sed -i "s|@REAL_CC@|${_REAL_CC}|g" "$_CXX_WRAPPER_DIR/cc" ++ chmod +x "$_CXX_WRAPPER_DIR/cc" ++ ++ # Point cmake at the wrappers via CXX/CC env vars (cmake reads these before ++ # probing PATH, and records them in CMakeCache.txt so subsequent cmake --build ++ # invocations also use the wrappers). Also prepend to PATH as a fallback. ++ export CXX="$_CXX_WRAPPER_DIR/c++" ++ export CC="$_CXX_WRAPPER_DIR/cc" ++ export PATH="$_CXX_WRAPPER_DIR:$PATH" ++ echo "Installed ppc64le compiler wrapper in $_CXX_WRAPPER_DIR (real CXX: $_REAL_CXX)" ++fi ++ ++# Delegate to the k-NN repository's own build script. ++exec bash scripts/build.sh "$@" +diff --git a/scripts/components/k-NN/integtest.sh b/scripts/components/k-NN/integtest.sh +index 9026e924..ced7bdc6 100755 +--- a/scripts/components/k-NN/integtest.sh ++++ b/scripts/components/k-NN/integtest.sh +@@ -102,6 +102,15 @@ fi + USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` + PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` + ++# On ppc64le, the kernel thread limit is tighter than on x86_64/arm64. ++# Gradle's default worker count (= CPU cores) combined with JVM GC threads ++# exhausts pthread resources, causing "pthread_create failed (EAGAIN)" and ++# preventing subprocesses such as javadoc from starting. Cap workers to 4 ++# to stay well within the limit on constrained ppc64le CI runners. ++if [ "$(uname -m)" = "ppc64le" ]; then ++ export GRADLE_OPTS="${GRADLE_OPTS} -Dorg.gradle.workers.max=4" ++fi ++ + # This will be added after 3.0.0 to k-NN repo directly + # As of now it is a temp measure to avoid building another Release Candidate + # while re-running test with a different configurations that is customizable +@@ -111,8 +120,8 @@ if [ "$OSTYPE" = "msys" ] || [ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "win32" ] + echo "Set new tests.path.repo to $REPO_PATH" + sed -i 's|^[[:space:]]\+task\.systemProperty\s*"tests\.path\.repo".*| task.systemProperty "tests.path.repo", System.getProperty("tests.path.repo", "${buildDir}/testSnapshotFolder")|' build.gradle + sed -i 's|^[[:space:]]\+systemProperty\s*"tests\.path\.repo".*| systemProperty "tests.path.repo", System.getProperty("tests.path.repo", "${buildDir}/testSnapshotFolder")|' build.gradle +- ./gradlew integTest -Dtests.path.repo="$REPO_PATH" -Dopensearch.version=$OPENSEARCH_VERSION -Dbuild.snapshot=$SNAPSHOT -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain ++ ./gradlew --console=plain integTest -Dtests.path.repo="$REPO_PATH" -Dopensearch.version=$OPENSEARCH_VERSION -Dbuild.snapshot=$SNAPSHOT -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain + + else +- ./gradlew integTest -Dopensearch.version=$OPENSEARCH_VERSION -Dbuild.snapshot=$SNAPSHOT -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain ++ ./gradlew --console=plain integTest -Dopensearch.version=$OPENSEARCH_VERSION -Dbuild.snapshot=$SNAPSHOT -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain + fi +diff --git a/scripts/components/notifications-core/build.sh b/scripts/components/notifications-core/build.sh +index 545661b4..4799f975 100644 +--- a/scripts/components/notifications-core/build.sh ++++ b/scripts/components/notifications-core/build.sh +@@ -67,14 +67,24 @@ fi + [[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER + [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT + [ -z "$OUTPUT" ] && OUTPUT=artifacts ++[ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m` + +-./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +-./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++# On ppc64le, the kernel thread limit is tighter than on x86_64/arm64. ++# Gradle's default worker count (= CPU cores) combined with JVM GC threads ++# exhausts pthread resources, causing "pthread_create failed (EAGAIN)" and ++# preventing subprocesses such as javadoc from starting. Cap workers to 4 ++# to stay well within the limit on constrained ppc64le CI runners. ++if [ "$ARCHITECTURE" = "ppc64le" ]; then ++ export GRADLE_OPTS="${GRADLE_OPTS} -Dorg.gradle.workers.max=4" ++fi ++ ++./gradlew --console=plain assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++./gradlew --console=plain publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + + mkdir -p ./$OUTPUT/plugins + notifCoreZipPath=$(ls core/build/distributions/ | grep .zip) + cp -v core/build/distributions/$notifCoreZipPath ./$OUTPUT/plugins + +-./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++./gradlew --console=plain publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + mkdir -p $OUTPUT/maven/org/opensearch/plugin + cp -r ./build/local-staging-repo/org/opensearch/plugin/opensearch-notifications-core $OUTPUT/maven/org/opensearch/plugin/ +diff --git a/scripts/components/notifications/build.sh b/scripts/components/notifications/build.sh +index c67f99d5..b4a536e8 100644 +--- a/scripts/components/notifications/build.sh ++++ b/scripts/components/notifications/build.sh +@@ -67,15 +67,25 @@ fi + [[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER + [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT + [ -z "$OUTPUT" ] && OUTPUT=artifacts ++[ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m` + +-./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +-./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++# On ppc64le, the kernel thread limit is tighter than on x86_64/arm64. ++# Gradle's default worker count (= CPU cores) combined with JVM GC threads ++# exhausts pthread resources, causing "pthread_create failed (EAGAIN)" and ++# preventing subprocesses such as javadoc from starting. Cap workers to 4 ++# to stay well within the limit on constrained ppc64le CI runners. ++if [ "$ARCHITECTURE" = "ppc64le" ]; then ++ export GRADLE_OPTS="${GRADLE_OPTS} -Dorg.gradle.workers.max=4" ++fi ++ ++./gradlew --console=plain assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++./gradlew --console=plain publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + + mkdir -p ./$OUTPUT/plugins + + notifCoreZipPath=$(ls notifications/build/distributions/ | grep .zip) + cp -v notifications/build/distributions/$notifCoreZipPath ./$OUTPUT/plugins + +-./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++./gradlew --console=plain publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + mkdir -p $OUTPUT/maven/org/opensearch/plugin + cp -r ./build/local-staging-repo/org/opensearch/plugin/notifications $OUTPUT/maven/org/opensearch/plugin/ +diff --git a/scripts/default/bwctest.sh b/scripts/default/bwctest.sh +index d1f687f5..eae8ad57 100755 +--- a/scripts/default/bwctest.sh ++++ b/scripts/default/bwctest.sh +@@ -35,4 +35,13 @@ while getopts ":h" arg; do + esac + done + +-./gradlew bwcTestSuite -Dtests.security.manager=false -PcustomDistributionDownloadType=bundle ++# On ppc64le, the kernel thread limit is tighter than on x86_64/arm64. ++# Gradle's default worker count (= CPU cores) combined with JVM GC threads ++# exhausts pthread resources, causing "pthread_create failed (EAGAIN)" and ++# preventing subprocesses such as javadoc from starting. Cap workers to 4 ++# to stay well within the limit on constrained ppc64le CI runners. ++if [ "$(uname -m)" = "ppc64le" ]; then ++ export GRADLE_OPTS="${GRADLE_OPTS} -Dorg.gradle.workers.max=4" ++fi ++ ++./gradlew --console=plain bwcTestSuite -Dtests.security.manager=false -PcustomDistributionDownloadType=bundle +diff --git a/scripts/default/integtest.sh b/scripts/default/integtest.sh +index 7a600d4c..c95880c7 100755 +--- a/scripts/default/integtest.sh ++++ b/scripts/default/integtest.sh +@@ -102,4 +102,13 @@ fi + USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` + PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` + +-./gradlew integTest -Dopensearch.version=$OPENSEARCH_VERSION -Dbuild.snapshot=$SNAPSHOT -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain ++# On ppc64le, the kernel thread limit is tighter than on x86_64/arm64. ++# Gradle's default worker count (= CPU cores) combined with JVM GC threads ++# exhausts pthread resources, causing "pthread_create failed (EAGAIN)" and ++# preventing subprocesses such as javadoc from starting. Cap workers to 4 ++# to stay well within the limit on constrained ppc64le CI runners. ++if [ "$(uname -m)" = "ppc64le" ]; then ++ export GRADLE_OPTS="${GRADLE_OPTS} -Dorg.gradle.workers.max=4" ++fi ++ ++./gradlew --console=plain integTest -Dopensearch.version=$OPENSEARCH_VERSION -Dbuild.snapshot=$SNAPSHOT -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.clustername="opensearch-integrationtest" -Dhttps=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain +diff --git a/scripts/default/opensearch/build.sh b/scripts/default/opensearch/build.sh +index b585940d..0aaf054b 100755 +--- a/scripts/default/opensearch/build.sh ++++ b/scripts/default/opensearch/build.sh +@@ -67,10 +67,20 @@ fi + [[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER + [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT + [ -z "$OUTPUT" ] && OUTPUT=artifacts ++[ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m` ++ ++# On ppc64le, the kernel thread limit is tighter than on x86_64/arm64. ++# Gradle's default worker count (= CPU cores) combined with JVM GC threads ++# exhausts pthread resources, causing "pthread_create failed (EAGAIN)" and ++# preventing subprocesses such as javadoc from starting. Cap workers to 4 ++# to stay well within the limit on constrained ppc64le CI runners. ++if [ "$ARCHITECTURE" = "ppc64le" ]; then ++ export GRADLE_OPTS="${GRADLE_OPTS} -Dorg.gradle.workers.max=4" ++fi + + mkdir -p $OUTPUT + +-./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++./gradlew --console=plain assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + + zipPath=$(find . -path \*build/distributions/*.zip) + distributions="$(dirname "${zipPath}")" +@@ -80,7 +90,7 @@ mkdir -p $OUTPUT/plugins + cp ${distributions}/*.zip ./$OUTPUT/plugins + + # Publish plugin zips to maven +-./gradlew publishPluginZipPublicationToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +-./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++./gradlew --console=plain publishPluginZipPublicationToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ++./gradlew --console=plain publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + mkdir -p $OUTPUT/maven/org/opensearch + cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch +diff --git a/security.patch b/security.patch +new file mode 100644 +index 00000000..b0660431 +--- /dev/null ++++ b/security.patch +@@ -0,0 +1,69 @@ ++diff --git a/build.gradle b/build.gradle ++index 0e148a8..58c45f5 100644 ++--- a/build.gradle +++++ b/build.gradle ++@@ -26,7 +26,7 @@ buildscript { ++ ++ common_utils_version = System.getProperty("common_utils.version", '3.2.0.0') ++ ++- kafka_version = '4.1.1' +++ kafka_version = '4.1.2' ++ open_saml_version = '5.1.6' ++ open_saml_shib_version = "9.1.4" ++ one_login_java_saml = '2.9.0' ++@@ -491,6 +491,16 @@ configurations { ++ force "io.netty:netty-handler:${versions.netty}" ++ force "io.netty:netty-transport:${versions.netty}" ++ force "io.netty:netty-transport-native-unix-common:${versions.netty}" +++ // CVE-2026-33870, CVE-2026-33871, CVE-2026-42584, CVE-2026-42587, CVE-2026-55831, +++ // CVE-2026-55833, CVE-2026-56745, CVE-2026-56819 — netty HTTP/HTTP2 codec fixes +++ force "io.netty:netty-codec-http:4.2.16.Final" +++ force "io.netty:netty-codec-http2:4.2.16.Final" +++ // CVE-2026-42582, CVE-2026-44892, CVE-2026-48748, CVE-2026-56816 — netty HTTP3 codec fixes +++ force "io.netty:netty-codec-http3:4.2.16.Final" +++ // CVE-2026-42583, CVE-2026-59901 — netty compression codec fixes +++ force "io.netty:netty-codec-compression:4.2.16.Final" +++ // CVE-2026-44894 — netty QUIC codec fixes +++ force "io.netty:netty-codec-classes-quic:4.2.15.Final" ++ force "com.github.luben:zstd-jni:${versions.zstd}" ++ force "org.xerial.snappy:snappy-java:1.1.10.8" ++ force "com.google.guava:guava:${guava_version}" ++@@ -502,8 +512,9 @@ configurations { ++ force "org.eclipse.platform:org.eclipse.core.runtime:3.34.100" ++ force "org.eclipse.platform:org.eclipse.equinox.common:3.20.300" ++ force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" ++- force "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" ++- force "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" +++ // CVE-2026-54399, CVE-2026-54428 — httpcore5 / httpcore5-h2 fixes +++ force "org.apache.httpcomponents.core5:httpcore5:5.4.3" +++ force "org.apache.httpcomponents.core5:httpcore5-h2:5.4.3" ++ force "org.codehaus.plexus:plexus-utils:3.6.0" ++ ++ // For integrationTest ++diff --git a/libs/opensaml/build.gradle b/libs/opensaml/build.gradle ++index ef72efb..f4b0448 100644 ++--- a/libs/opensaml/build.gradle +++++ b/libs/opensaml/build.gradle ++@@ -31,6 +31,15 @@ configurations.all { ++ force "org.bouncycastle:bcpkix-jdk18on:1.83" ++ force "org.bouncycastle:bcprov-jdk18on:1.83" ++ force "org.apache.commons:commons-lang3:${versions.commonslang}" +++ // CVE-2026-54399, CVE-2026-54428 — httpcore5 / httpcore5-h2 bundled in opensaml shadow jar +++ force "org.apache.httpcomponents.core5:httpcore5:5.4.3" +++ force "org.apache.httpcomponents.core5:httpcore5-h2:5.4.3" +++ // CVE-2026-54512, CVE-2026-54513, GHSA-r7wm-3cxj-wff9 — jackson bundled in opensaml shadow jar +++ // Must match jackson-annotations 2.21 / jackson-databind 2.21.4 used by OpenSearch core to avoid +++ // NoClassDefFoundError: JsonSerializeAs (added in annotations 2.21, required by databind 2.21.x). +++ force "com.fasterxml.jackson.core:jackson-databind:2.21.4" +++ force "com.fasterxml.jackson.core:jackson-core:2.21.4" +++ force "com.fasterxml.jackson.core:jackson-annotations:2.21" ++ } ++ } ++ ++@@ -81,5 +90,6 @@ tasks.shadowJar { ++ exclude 'META-INF/maven/commons-io/commons-io/**' ++ exclude 'META-INF/maven/org.apache.commons/commons-lang3/**' ++ exclude 'META-INF/versions/**/org/bouncycastle/**' +++ exclude 'META-INF/versions/**/com/**' ++ exclude 'META-INF/services/org.opensaml.security.crypto.ec.NamedCurve' ++ } +diff --git a/src/build_workflow/build_args.py b/src/build_workflow/build_args.py +index 1ad26503..76b86e0a 100644 +--- a/src/build_workflow/build_args.py ++++ b/src/build_workflow/build_args.py +@@ -16,6 +16,7 @@ class BuildArgs: + SUPPORTED_ARCHITECTURES = [ + "x64", + "arm64", ++ "ppc64le", + ] + SUPPORTED_DISTRIBUTIONS = ["tar", "zip", "rpm", "deb"] + +diff --git a/src/build_workflow/builder_from_source.py b/src/build_workflow/builder_from_source.py +index 53c47bbb..8b17c4ba 100644 +--- a/src/build_workflow/builder_from_source.py ++++ b/src/build_workflow/builder_from_source.py +@@ -5,6 +5,8 @@ + # this file be licensed under the Apache-2.0 license or a + # compatible open source license. + ++import glob ++import logging + import os + + from build_workflow.build_recorder import BuildRecorder +@@ -12,6 +14,16 @@ from build_workflow.builder import Builder + from git.git_repository import GitRepository + from paths.script_finder import ScriptFinder + ++# Resolve patch file paths as absolute paths relative to the repo root (two levels up from this ++# source file: src/build_workflow/ -> src/ -> repo root). Using absolute paths ensures that ++# both os.path.isfile() and `git apply` work correctly regardless of the process's current ++# working directory, which changes to a temp dir early in run_build.py via chdir=True. ++_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) ++OPENSEARCH_PATCH_FILE = os.path.join(_REPO_ROOT, "opensearch.patch") ++COMMON_UTILS_PATCH_FILE = os.path.join(_REPO_ROOT, "common-utils.patch") ++KNN_PATCH_FILE = os.path.join(_REPO_ROOT, "knn.patch") ++SECURITY_PATCH_FILE = os.path.join(_REPO_ROOT, "security.patch") ++ + """ + This class is responsible for executing the build for a component and passing the results to a build recorder. + It will notify the build recorder of build information such as repository and git ref, and any artifacts generated by the build. +@@ -27,7 +39,113 @@ class BuilderFromSource(Builder): + os.path.join(work_dir, self.component.name), + self.component.working_directory, + ) ++ ++ # Apply OpenSearch patch if building OpenSearch ++ if self.component.name == "OpenSearch": ++ if os.path.isfile(OPENSEARCH_PATCH_FILE): ++ logging.info(f"Applying patch {OPENSEARCH_PATCH_FILE} to {self.component.name}") ++ self.git_repo.execute(f"git apply {OPENSEARCH_PATCH_FILE}") ++ logging.info(f"Successfully applied patch to {self.component.name}") ++ else: ++ logging.warning(f"Patch file not found: {OPENSEARCH_PATCH_FILE}") ++ ++ # Apply common-utils patch if building common-utils ++ if self.component.name == "common-utils": ++ if os.path.isfile(COMMON_UTILS_PATCH_FILE): ++ logging.info(f"Applying patch {COMMON_UTILS_PATCH_FILE} to {self.component.name}") ++ self.git_repo.execute(f"git apply {COMMON_UTILS_PATCH_FILE}") ++ logging.info(f"Successfully applied patch to {self.component.name}") ++ else: ++ logging.warning(f"Patch file not found: {COMMON_UTILS_PATCH_FILE}") ++ ++ # Apply k-NN patch if building k-NN ++ if self.component.name == "k-NN": ++ if os.path.isfile(KNN_PATCH_FILE): ++ logging.info(f"Applying patch {KNN_PATCH_FILE} to {self.component.name}") ++ self.git_repo.execute(f"git apply {KNN_PATCH_FILE}") ++ logging.info(f"Successfully applied patch to {self.component.name}") ++ else: ++ logging.warning(f"Patch file not found: {KNN_PATCH_FILE}") ++ ++ # Apply security patch if building security ++ if self.component.name == "security": ++ if os.path.isfile(SECURITY_PATCH_FILE): ++ logging.info(f"Applying patch {SECURITY_PATCH_FILE} to {self.component.name}") ++ self.git_repo.execute(f"git apply {SECURITY_PATCH_FILE}") ++ logging.info(f"Successfully applied patch to {self.component.name}") ++ else: ++ logging.warning(f"Patch file not found: {SECURITY_PATCH_FILE}") ++ ++ # Apply ppc64le fix for all Gradle-based components ++ self._apply_ppc64le_gradle_fix() ++ ++ def _apply_ppc64le_gradle_fix(self) -> None: ++ """ ++ Apply ppc64le architecture fix for Gradle builds. ++ ++ Two parts: ++ 1. gradle.properties – disables native-platform and rich console output. ++ 2. ~/.gradle/init.d/ppc64le-kotlin-fix.gradle – an init script that spoofs ++ os.arch to x86_64 before plugin classloading occurs. This is the only ++ reliable fix for the Kotlin 2.x crash: NativeCompilerDownloader. ++ calls HostManager.hostArch() at configuration time and throws ++ TargetSupportException: Unknown hardware platform: ppc64le before any ++ gradle.properties value is ever read. The init script condition means it ++ is a no-op on all non-ppc64le hosts. ++ """ ++ gradle_properties_path = os.path.join(self.git_repo.working_directory, "gradle.properties") ++ ++ # Check if this is a Gradle project (has gradlew or build.gradle) ++ has_gradlew = os.path.isfile(os.path.join(self.git_repo.working_directory, "gradlew")) ++ has_build_gradle = os.path.isfile(os.path.join(self.git_repo.working_directory, "build.gradle")) + ++ if not (has_gradlew or has_build_gradle): ++ logging.debug(f"Skipping ppc64le Gradle fix for {self.component.name} - not a Gradle project") ++ return ++ ++ gradle_properties_content = """# Disable native platform support for ppc64le architecture compatibility ++# The native-platform library doesn't support ppc64le, so we fall back to pure Java implementations ++org.gradle.native=false ++ ++# Use plain console output (no rich formatting that requires native platform) ++org.gradle.console=plain ++""" ++ ++ # If gradle.properties already exists, append our settings ++ if os.path.isfile(gradle_properties_path): ++ logging.info(f"Appending ppc64le fix to existing gradle.properties for {self.component.name}") ++ with open(gradle_properties_path, 'a') as f: ++ f.write("\n" + gradle_properties_content) ++ else: ++ logging.info(f"Creating gradle.properties with ppc64le fix for {self.component.name}") ++ with open(gradle_properties_path, 'w') as f: ++ f.write(gradle_properties_content) ++ ++ # The Kotlin 2.x NativeCompilerDownloader. crash on ppc64le ++ # affects any component that uses Kotlin 2.x (e.g. cross-cluster-replication, ++ # opensearch-observability). Install a Gradle init script that spoofs ++ # os.arch to x86_64 before plugin classloading occurs — this is the only ++ # fix that works because the crash happens before any gradle.properties ++ # value is ever read. The condition in the script makes it a no-op on ++ # non-ppc64le hosts so it is safe to install for all Kotlin-using components. ++ KOTLIN_COMPONENTS = {"cross-cluster-replication", "opensearch-observability"} ++ if self.component.name in KOTLIN_COMPONENTS: ++ gradle_init_d = os.path.join(os.path.expanduser("~"), ".gradle", "init.d") ++ os.makedirs(gradle_init_d, exist_ok=True) ++ init_script_path = os.path.join(gradle_init_d, "ppc64le-kotlin-fix.gradle") ++ init_script_content = """\ ++// Spoof os.arch to x86_64 on ppc64le hosts so that the Kotlin 2.x Gradle plugin ++// does not crash during static class initialisation (NativeCompilerDownloader. ++// calls HostManager.hostArch() which throws TargetSupportException for unknown ++// hardware platforms). The condition makes this a no-op on all other architectures. ++if (System.getProperty("os.arch") == "ppc64le") { ++ System.setProperty("os.arch", "x86_64") ++} ++""" ++ logging.info(f"Writing Gradle init script for ppc64le Kotlin fix: {init_script_path}") ++ with open(init_script_path, 'w') as f: ++ f.write(init_script_content) ++ + def build(self, build_recorder: BuildRecorder) -> None: + + # List of components whose build scripts support `-d` parameter +@@ -53,7 +171,7 @@ class BuilderFromSource(Builder): + ) + ) + +- self.git_repo.execute(build_command) ++ self.git_repo.execute(build_command) + build_recorder.record_component(self.component.name, self.git_repo) + + def export_artifacts(self, build_recorder: BuildRecorder) -> None: +@@ -63,4 +181,4 @@ class BuilderFromSource(Builder): + for file_name in files: + absolute_path = os.path.join(dir, file_name) + relative_path = os.path.relpath(absolute_path, artifacts_path) +- build_recorder.record_artifact(self.component.name, artifact_type, relative_path, absolute_path) ++ build_recorder.record_artifact(self.component.name, artifact_type, relative_path, absolute_path) +\ No newline at end of file +diff --git a/src/ci_workflow/ci_check_gradle_dependencies.py b/src/ci_workflow/ci_check_gradle_dependencies.py +index 5295b85c..b4e1949a 100644 +--- a/src/ci_workflow/ci_check_gradle_dependencies.py ++++ b/src/ci_workflow/ci_check_gradle_dependencies.py +@@ -27,7 +27,7 @@ class CiCheckGradleDependencies(CiCheckSource): + filter( + None, + [ +- f"./gradlew {self.gradle_project or ''}:dependencies", ++ f"./gradlew --console=plain {self.gradle_project or ''}:dependencies", + f"-Dopensearch.version={self.target.opensearch_version}", + f"-Dbuild.snapshot={str(self.target.snapshot).lower()}", + f"-Dbuild.version_qualifier={str(self.target.qualifier)}" if self.target.qualifier else None, +diff --git a/src/ci_workflow/ci_check_gradle_properties.py b/src/ci_workflow/ci_check_gradle_properties.py +index 4eb073c7..19bb99b1 100644 +--- a/src/ci_workflow/ci_check_gradle_properties.py ++++ b/src/ci_workflow/ci_check_gradle_properties.py +@@ -24,7 +24,7 @@ class CiCheckGradleProperties(CiCheckSource): + filter( + None, + [ +- "./gradlew properties", ++ "./gradlew --console=plain properties", + f"-Dopensearch.version={self.target.opensearch_version}", + f"-Dbuild.snapshot={str(self.target.snapshot).lower()}", + f"-Dbuild.version_qualifier={str(self.target.qualifier)}" if self.target.qualifier else None, +diff --git a/src/ci_workflow/ci_check_gradle_publish_to_maven_local.py b/src/ci_workflow/ci_check_gradle_publish_to_maven_local.py +index dfdc6fab..2ba2e07b 100644 +--- a/src/ci_workflow/ci_check_gradle_publish_to_maven_local.py ++++ b/src/ci_workflow/ci_check_gradle_publish_to_maven_local.py +@@ -14,7 +14,7 @@ class CiCheckGradlePublishToMavenLocal(CiCheckSource): + filter( + None, + [ +- "./gradlew publishToMavenLocal", ++ "./gradlew --console=plain publishToMavenLocal", + f"-Dopensearch.version={self.target.opensearch_version}", + f"-Dbuild.snapshot={str(self.target.snapshot).lower()}", + f"-Dbuild.version_qualifier={str(self.target.qualifier)}" if self.target.qualifier else None, +diff --git a/src/manifests_workflow/component_opensearch.py b/src/manifests_workflow/component_opensearch.py +index c382e106..71d5e2af 100644 +--- a/src/manifests_workflow/component_opensearch.py ++++ b/src/manifests_workflow/component_opensearch.py +@@ -64,6 +64,6 @@ class ComponentOpenSearch(Component): + + @classmethod + def gradle_cmd(self, target: str, props: dict = {}) -> str: +- cmd = [f"./gradlew {target}"] ++ cmd = [f"./gradlew --console=plain {target}"] + cmd.extend([f"-D{k}={v}" for k, v in props.items()]) + return " ".join(cmd) +diff --git a/tests/tests_ci_workflow/test_ci_check_gradle_dependencies.py b/tests/tests_ci_workflow/test_ci_check_gradle_dependencies.py +index 808fbd38..ed4d0f4e 100644 +--- a/tests/tests_ci_workflow/test_ci_check_gradle_dependencies.py ++++ b/tests/tests_ci_workflow/test_ci_check_gradle_dependencies.py +@@ -33,34 +33,34 @@ class TestCiCheckGradleDependencies(unittest.TestCase): + def test_executes_gradle_dependencies(self) -> None: + check = self.__mock_dependencies() + output = unittest.mock.create_autospec(check.git_repo.output) +- output.assert_called_once_with('./gradlew :dependencies -Dopensearch.version=1.1.0 -Dbuild.snapshot=false --configuration compileOnly | grep -e "---"') ++ output.assert_called_once_with('./gradlew --console=plain :dependencies -Dopensearch.version=1.1.0 -Dbuild.snapshot=false --configuration compileOnly | grep -e "---"') + + def test_executes_gradle_dependencies_snapshot(self) -> None: + check = self.__mock_dependencies(snapshot=True) + output = unittest.mock.create_autospec(check.git_repo.output) + output.assert_called_once_with( +- './gradlew :dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"' ++ './gradlew --console=plain :dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"' + ) + + def test_executes_gradle_dependencies_qualifier_snapshot(self) -> None: + check = self.__mock_dependencies(qualifier="alpha1", snapshot=True) + output = unittest.mock.create_autospec(check.git_repo.output) + output.assert_called_once_with( +- './gradlew :dependencies -Dopensearch.version=1.1.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"' ++ './gradlew --console=plain :dependencies -Dopensearch.version=1.1.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"' + ) + + def test_executes_gradle_dependencies_project(self) -> None: + check = self.__mock_dependencies(snapshot=True, gradle_project="project") + output = unittest.mock.create_autospec(check.git_repo.output) + output.assert_called_once_with( +- './gradlew project:dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"' ++ './gradlew --console=plain project:dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"' + ) + + def test_executes_gradle_dependencies_project_qualifier(self) -> None: + check = self.__mock_dependencies(qualifier="alpha1", snapshot=True, gradle_project="project") + output = unittest.mock.create_autospec(check.git_repo.output) + output.assert_called_once_with( +- './gradlew project:dependencies -Dopensearch.version=1.1.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"' ++ './gradlew --console=plain project:dependencies -Dopensearch.version=1.1.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"' + ) + + def test_loads_tree(self) -> None: +diff --git a/tests/tests_ci_workflow/test_ci_check_gradle_dependencies_opensearch.py b/tests/tests_ci_workflow/test_ci_check_gradle_dependencies_opensearch.py +index 46c54e85..db4d6016 100644 +--- a/tests/tests_ci_workflow/test_ci_check_gradle_dependencies_opensearch.py ++++ b/tests/tests_ci_workflow/test_ci_check_gradle_dependencies_opensearch.py +@@ -59,7 +59,7 @@ class TestCiCheckGradleDependenciesOpenSearchVersion(unittest.TestCase): + ) + output = unittest.mock.create_autospec(check.git_repo.output) + output.assert_called_once_with( +- './gradlew :dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"' ++ './gradlew --console=plain :dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"' + ) + + def test_executes_gradle_command_qualifier(self) -> None: +@@ -71,7 +71,7 @@ class TestCiCheckGradleDependenciesOpenSearchVersion(unittest.TestCase): + ) + output = unittest.mock.create_autospec(check.git_repo.output) + output.assert_called_once_with( +- './gradlew :dependencies -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"' ++ './gradlew --console=plain :dependencies -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"' + ) + + def test_executes_gradle_command_with_arg(self) -> None: +@@ -83,7 +83,7 @@ class TestCiCheckGradleDependenciesOpenSearchVersion(unittest.TestCase): + ) + output = unittest.mock.create_autospec(check.git_repo.output) + output.assert_called_once_with( +- './gradlew plugin:dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"' ++ './gradlew --console=plain plugin:dependencies -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true --configuration compileOnly | grep -e "---"' + ) + + def test_executes_gradle_command_qualifier_with_arg(self) -> None: +@@ -95,5 +95,5 @@ class TestCiCheckGradleDependenciesOpenSearchVersion(unittest.TestCase): + ) + output = unittest.mock.create_autospec(check.git_repo.output) + output.assert_called_once_with( +- './gradlew plugin:dependencies -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"' ++ './gradlew --console=plain plugin:dependencies -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1 --configuration compileOnly | grep -e "---"' + ) +diff --git a/tests/tests_ci_workflow/test_ci_check_gradle_properties.py b/tests/tests_ci_workflow/test_ci_check_gradle_properties.py +index 16c3a918..5d4fa4cc 100644 +--- a/tests/tests_ci_workflow/test_ci_check_gradle_properties.py ++++ b/tests/tests_ci_workflow/test_ci_check_gradle_properties.py +@@ -27,7 +27,7 @@ class TestCiCheckGradleProperties(unittest.TestCase): + target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=False), + ) + +- git_repo.output.assert_called_once_with("./gradlew properties -Dopensearch.version=1.1.0 -Dbuild.snapshot=false") ++ git_repo.output.assert_called_once_with("./gradlew --console=plain properties -Dopensearch.version=1.1.0 -Dbuild.snapshot=false") + + def test_executes_gradle_properties_snapshot(self) -> None: + git_repo = MagicMock() +@@ -39,7 +39,7 @@ class TestCiCheckGradleProperties(unittest.TestCase): + target=CiTarget(version="1.1.0", name="opensearch", qualifier=None, snapshot=True), + ) + +- git_repo.output.assert_called_once_with("./gradlew properties -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true") ++ git_repo.output.assert_called_once_with("./gradlew --console=plain properties -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true") + + def test_executes_gradle_properties_qualifier_snapshot(self) -> None: + git_repo = MagicMock() +@@ -51,4 +51,4 @@ class TestCiCheckGradleProperties(unittest.TestCase): + target=CiTarget(version="2.0.0", name="opensearch", qualifier="alpha1", snapshot=True), + ) + +- git_repo.output.assert_called_once_with("./gradlew properties -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1") ++ git_repo.output.assert_called_once_with("./gradlew --console=plain properties -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1") +diff --git a/tests/tests_ci_workflow/test_ci_check_gradle_publish_to_maven_local.py b/tests/tests_ci_workflow/test_ci_check_gradle_publish_to_maven_local.py +index 0991e0f9..847d1b8f 100644 +--- a/tests/tests_ci_workflow/test_ci_check_gradle_publish_to_maven_local.py ++++ b/tests/tests_ci_workflow/test_ci_check_gradle_publish_to_maven_local.py +@@ -21,7 +21,7 @@ class TestCiCheckGradlePublishToMavenLocal(unittest.TestCase): + ) + check.check() + exec_command = unittest.mock.create_autospec(check.git_repo.execute) +- exec_command.assert_called_once_with("./gradlew publishToMavenLocal -Dopensearch.version=1.1.0 -Dbuild.snapshot=false") ++ exec_command.assert_called_once_with("./gradlew --console=plain publishToMavenLocal -Dopensearch.version=1.1.0 -Dbuild.snapshot=false") + + def test_executes_gradle_command_snapshot(self) -> None: + check = CiCheckGradlePublishToMavenLocal( +@@ -31,7 +31,7 @@ class TestCiCheckGradlePublishToMavenLocal(unittest.TestCase): + ) + check.check() + exec_command = unittest.mock.create_autospec(check.git_repo.execute) +- exec_command.assert_called_once_with("./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true") ++ exec_command.assert_called_once_with("./gradlew --console=plain publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT -Dbuild.snapshot=true") + + def test_executes_gradle_command_qualifier_snapshot(self) -> None: + check = CiCheckGradlePublishToMavenLocal( +@@ -41,4 +41,4 @@ class TestCiCheckGradlePublishToMavenLocal(unittest.TestCase): + ) + check.check() + exec_command = unittest.mock.create_autospec(check.git_repo.execute) +- exec_command.assert_called_once_with("./gradlew publishToMavenLocal -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1") ++ exec_command.assert_called_once_with("./gradlew --console=plain publishToMavenLocal -Dopensearch.version=2.0.0-alpha1-SNAPSHOT -Dbuild.snapshot=true -Dbuild.version_qualifier=alpha1") +diff --git a/tests/tests_manifests_workflow/test_component_opensearch.py b/tests/tests_manifests_workflow/test_component_opensearch.py +index 267d8a84..8f7a5721 100644 +--- a/tests/tests_manifests_workflow/test_component_opensearch.py ++++ b/tests/tests_manifests_workflow/test_component_opensearch.py +@@ -42,16 +42,16 @@ class TestComponentOpenSearch(unittest.TestCase): + ) + + def test_gradle_cmd_target(self) -> None: +- self.assertEqual(ComponentOpenSearch.gradle_cmd("properties"), "./gradlew properties") ++ self.assertEqual(ComponentOpenSearch.gradle_cmd("properties"), "./gradlew --console=plain properties") + + def test_gradle_cmd_prop(self) -> None: + self.assertEqual( + ComponentOpenSearch.gradle_cmd("properties", {"build.snapshot": "false"}), +- "./gradlew properties -Dbuild.snapshot=false", ++ "./gradlew --console=plain properties -Dbuild.snapshot=false", + ) + + def test_gradle_cmd_props(self) -> None: + self.assertEqual( + ComponentOpenSearch.gradle_cmd("properties", {"build.snapshot": "false", "opensearch.version": "1.0"}), +- "./gradlew properties -Dbuild.snapshot=false -Dopensearch.version=1.0", ++ "./gradlew --console=plain properties -Dbuild.snapshot=false -Dopensearch.version=1.0", + ) +diff --git a/tools/vulnerability-scan/wss-unified-agent.config b/tools/vulnerability-scan/wss-unified-agent.config +index 56bb7518..98129aca 100644 +--- a/tools/vulnerability-scan/wss-unified-agent.config ++++ b/tools/vulnerability-scan/wss-unified-agent.config +@@ -81,7 +81,7 @@ followSymbolicLinks=true + gradle.resolveDependencies=true + gradle.aggregateModules=true + gradle.preferredEnvironment=gradle +-#gradle.wrapperPath=./gradlew ++#gradle.wrapperPath=./gradlew --console=plain + maven.resolveDependencies=true + maven.runPreStep=true + maven.aggregateModules=true +-- +2.51.2 +