From c848f426d5dd0eeafa024ed6ad06aa06b061f78f Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:40:31 +0900 Subject: [PATCH 1/3] Replace MinIO with RustFS for S3 compat CI --- .github/scripts/generate_ci_config.sh | 10 +- .github/scripts/setup_minio_buckets.sh | 73 --------- .github/scripts/setup_rdm.sh | 8 +- .../{setup_minio.sh => setup_rustfs.sh} | 50 +++---- .github/scripts/setup_rustfs_buckets.sh | 73 +++++++++ .github/workflows/e2e-test.yml | 140 +++++++----------- 6 files changed, 160 insertions(+), 194 deletions(-) delete mode 100755 .github/scripts/setup_minio_buckets.sh rename .github/scripts/{setup_minio.sh => setup_rustfs.sh} (71%) create mode 100755 .github/scripts/setup_rustfs_buckets.sh diff --git a/.github/scripts/generate_ci_config.sh b/.github/scripts/generate_ci_config.sh index 86d960c..d643f3c 100755 --- a/.github/scripts/generate_ci_config.sh +++ b/.github/scripts/generate_ci_config.sh @@ -3,7 +3,7 @@ set -xeuo pipefail if [[ $# -lt 2 ]]; then cat >&2 <<'USAGE' -Usage: generate_ci_config.sh [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] +Usage: generate_ci_config.sh [--rustfs] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] USAGE exit 1 fi @@ -11,7 +11,7 @@ fi OUTPUT=$1 BASE_CONFIG=$2 shift 2 -MINIO=false +RUSTFS=false JUPYTERHUB=false WEKO=false FLOWABLE=false @@ -21,8 +21,8 @@ WIKI=false for arg in "$@"; do case "$arg" in - --minio) - MINIO=true + --rustfs) + RUSTFS=true ;; --jupyterhub) JUPYTERHUB=true @@ -52,7 +52,7 @@ done cp "${BASE_CONFIG}" "${OUTPUT}" -if [[ "${MINIO}" == "true" ]]; then +if [[ "${RUSTFS}" == "true" ]]; then if [[ -z "${S3COMPAT_ACCESS_KEY_1:-}" || -z "${S3COMPAT_SECRET_KEY_1:-}" ]]; then echo "S3 compat credentials are not set" >&2 exit 1 diff --git a/.github/scripts/setup_minio_buckets.sh b/.github/scripts/setup_minio_buckets.sh deleted file mode 100755 index 479d0b8..0000000 --- a/.github/scripts/setup_minio_buckets.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -xeuo pipefail - -# Usage: -# ./setup_minio_buckets.sh \ -# - -if [[ $# -ne 11 ]]; then - cat >&2 <<'EOF' -Usage: setup_minio_buckets.sh \ - -EOF - exit 1 -fi - -RDM_ROOT=$1 -ALIAS=$2 -ENDPOINT=$3 -ROOT_USER=$4 -ROOT_PASSWORD=$5 -ACCESS_KEY_1=$6 -SECRET_KEY_1=$7 -BUCKET_NAME_1=$8 -ACCESS_KEY_2=$9 -SECRET_KEY_2=${10} -BUCKET_NAME_2=${11} - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -MINIO_VERIFY_LARGE_UPLOAD=${MINIO_VERIFY_LARGE_UPLOAD:-false} -MINIO_VERIFY_LARGE_UPLOAD_SIZE_MB=${MINIO_VERIFY_LARGE_UPLOAD_SIZE_MB:-130} -MINIO_VERIFY_LARGE_UPLOAD_KEY=${MINIO_VERIFY_LARGE_UPLOAD_KEY:-diagnostics/minio-large-upload.bin} - -if [[ ! -d "${RDM_ROOT}" ]]; then - echo "RDM root directory not found: ${RDM_ROOT}" >&2 - exit 1 -fi - -pushd "${RDM_ROOT}" >/dev/null -source "${SCRIPT_DIR}/lib/wait_for_service.sh" - -SERVICE_NAME="MinIO" -CHECK_COMMAND="docker-compose exec -T minio /bin/sh -c 'curl -fs http://localhost:9000/minio/health/ready'" -TIMEOUT=180 -INTERVAL=5 -wait_for_service - -docker-compose run --rm --entrypoint /bin/sh minio-mc <&2 - dd if=/dev/zero of=/tmp/minio-large-upload-test.bin bs=1M count=${MINIO_VERIFY_LARGE_UPLOAD_SIZE_MB} - mc cp /tmp/minio-large-upload-test.bin "${ALIAS}/${BUCKET_NAME_1}/${MINIO_VERIFY_LARGE_UPLOAD_KEY}" - mc stat "${ALIAS}/${BUCKET_NAME_1}/${MINIO_VERIFY_LARGE_UPLOAD_KEY}" - rm -f /tmp/minio-large-upload-test.bin -fi -EOF - -popd >/dev/null - -echo "MinIO buckets and users have been initialized" diff --git a/.github/scripts/setup_rdm.sh b/.github/scripts/setup_rdm.sh index 2111510..7975c91 100644 --- a/.github/scripts/setup_rdm.sh +++ b/.github/scripts/setup_rdm.sh @@ -85,8 +85,8 @@ start_rdm_services() { SERVICES="kaken_elasticsearch $SERVICES" - if [ "${MINIO_ENABLED:-false}" = "true" ]; then - SERVICES="$SERVICES minio" + if [ "${RUSTFS_ENABLED:-false}" = "true" ]; then + SERVICES="$SERVICES rustfs" fi echo "Starting services: $SERVICES" @@ -336,10 +336,10 @@ EOL echo "" >> .docker-compose.wb.env echo "SERVER_CONFIG_DOMAIN=http://192.168.168.167:7777" >> .docker-compose.wb.env - if [ "${MINIO_ENABLED:-false}" = "true" ]; then + if [ "${RUSTFS_ENABLED:-false}" = "true" ]; then local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - bash "${script_dir}/setup_minio.sh" apply "${PWD}" + bash "${script_dir}/setup_rustfs.sh" apply "${PWD}" fi if [ "${WEKO_ENABLED:-false}" = "true" ] || [ "${MIBYO_DB_ENABLED:-false}" = "true" ]; then diff --git a/.github/scripts/setup_minio.sh b/.github/scripts/setup_rustfs.sh similarity index 71% rename from .github/scripts/setup_minio.sh rename to .github/scripts/setup_rustfs.sh index 685cf82..503acf0 100755 --- a/.github/scripts/setup_minio.sh +++ b/.github/scripts/setup_rustfs.sh @@ -2,10 +2,10 @@ set -xeuo pipefail # Usage: -# ./setup_minio.sh apply +# ./setup_rustfs.sh apply # # Generates docker-compose override entries and s3compat settings -# required for MinIO usage within the E2E environment. +# required for RustFS usage within the E2E environment. COMMAND=${1:-} RDM_ROOT=${2:-} @@ -29,28 +29,24 @@ if [[ ! -d "${RDM_ROOT}" ]]; then exit 1 fi -MINIO_IMAGE_DEFAULT=${MINIO_IMAGE:-minio/minio:latest} -MINIO_MC_IMAGE_DEFAULT=${MINIO_MC_IMAGE:-minio/mc:latest} +RUSTFS_IMAGE_DEFAULT=${RUSTFS_IMAGE:-rustfs/rustfs:latest} +RUSTFS_RC_IMAGE_DEFAULT=${RUSTFS_RC_IMAGE:-rustfs/rc:latest} -MINIO_DOCKER_SNIPPET=$(cat < "${compose_override}" fi -if ! grep -q '^ minio:' "${compose_override}"; then - printf '\n%s\n' "${MINIO_DOCKER_SNIPPET}" >> "${compose_override}" +if ! grep -q '^ rustfs:' "${compose_override}"; then + printf '\n%s\n' "${RUSTFS_DOCKER_SNIPPET}" >> "${compose_override}" fi settings_json="${RDM_ROOT}/addons/s3compat/static/settings.json" @@ -81,8 +77,8 @@ src, dst = sys.argv[1:3] with open(src) as f: data = json.load(f) -service_name = "MinIO (CI)" -host = "minio:9000" +service_name = "RustFS (CI)" +host = "rustfs:9000" available_services = data.setdefault("availableServices", []) @@ -109,7 +105,7 @@ PY mv "${tmp_json}" "${settings_json}" -# Also register MinIO service for s3compatsigv4 addon if available +# Also register RustFS service for s3compatsigv4 addon if available sigv4_settings_json="${RDM_ROOT}/addons/s3compatsigv4/static/settings.json" if [[ -f "${sigv4_settings_json}" ]]; then @@ -123,8 +119,8 @@ src, dst = sys.argv[1:3] with open(src) as f: data = json.load(f) -service_name = "MinIO (CI)" -host = "minio:9000" +service_name = "RustFS (CI)" +host = "rustfs:9000" available_services = data.setdefault("availableServices", []) @@ -150,9 +146,9 @@ with open(dst, "w") as f: PY mv "${tmp_json_sigv4}" "${sigv4_settings_json}" - echo "MinIO configuration applied for s3compatsigv4" + echo "RustFS configuration applied for s3compatsigv4" else echo "WARNING: s3compatsigv4 settings.json not found: ${sigv4_settings_json} (skipping)" fi -echo "MinIO configuration applied" +echo "RustFS configuration applied" diff --git a/.github/scripts/setup_rustfs_buckets.sh b/.github/scripts/setup_rustfs_buckets.sh new file mode 100755 index 0000000..f7f65ec --- /dev/null +++ b/.github/scripts/setup_rustfs_buckets.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -xeuo pipefail + +# Usage: +# ./setup_rustfs_buckets.sh \ +# + +if [[ $# -ne 11 ]]; then + cat >&2 <<'EOF' +Usage: setup_rustfs_buckets.sh \ + +EOF + exit 1 +fi + +RDM_ROOT=$1 +ALIAS=$2 +ENDPOINT=$3 +ROOT_USER=$4 +ROOT_PASSWORD=$5 +ACCESS_KEY_1=$6 +SECRET_KEY_1=$7 +BUCKET_NAME_1=$8 +ACCESS_KEY_2=$9 +SECRET_KEY_2=${10} +BUCKET_NAME_2=${11} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RUSTFS_VERIFY_LARGE_UPLOAD=${RUSTFS_VERIFY_LARGE_UPLOAD:-false} +RUSTFS_VERIFY_LARGE_UPLOAD_SIZE_MB=${RUSTFS_VERIFY_LARGE_UPLOAD_SIZE_MB:-130} +RUSTFS_VERIFY_LARGE_UPLOAD_KEY=${RUSTFS_VERIFY_LARGE_UPLOAD_KEY:-diagnostics/rustfs-large-upload.bin} + +if [[ ! -d "${RDM_ROOT}" ]]; then + echo "RDM root directory not found: ${RDM_ROOT}" >&2 + exit 1 +fi + +pushd "${RDM_ROOT}" >/dev/null +source "${SCRIPT_DIR}/lib/wait_for_service.sh" + +SERVICE_NAME="RustFS" +CHECK_COMMAND="docker-compose exec -T rustfs /bin/sh -c 'curl -fs http://localhost:9000/health/ready'" +TIMEOUT=180 +INTERVAL=5 +wait_for_service + +docker-compose run --rm --entrypoint /bin/sh rustfs-rc <&2 + dd if=/dev/zero of=/tmp/rustfs-large-upload-test.bin bs=1M count=${RUSTFS_VERIFY_LARGE_UPLOAD_SIZE_MB} + rc object copy /tmp/rustfs-large-upload-test.bin "${ALIAS}/${BUCKET_NAME_1}/${RUSTFS_VERIFY_LARGE_UPLOAD_KEY}" + rc object stat "${ALIAS}/${BUCKET_NAME_1}/${RUSTFS_VERIFY_LARGE_UPLOAD_KEY}" + rm -f /tmp/rustfs-large-upload-test.bin +fi +EOF + +popd >/dev/null + +echo "RustFS buckets and users have been initialized" diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 8897a8a..e1b7b5d 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -118,7 +118,7 @@ jobs: skip_admin: true skip_metadata: false skip_autofill: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -133,7 +133,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -146,7 +146,7 @@ jobs: skip_metadata: true skip_default_storage: true skip_login: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -189,7 +189,7 @@ jobs: テスト手順-管理者機能-アナウンス.ipynb テスト手順-管理者機能-証跡管理.ipynb テスト手順-管理者機能-NIIストレージのクォータ.ipynb - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -211,7 +211,7 @@ jobs: exclude_notebooks: | テスト手順-WEKOアドオン-メタデータ送信-書誌情報.ipynb テスト手順-WEKOアドオン-メタデータ送信-書誌情報＋根拠データ.ipynb - minio_enabled: false + rustfs_enabled: false weko_enabled: true mibyo_db_enabled: false weko_test_mode: direct @@ -222,8 +222,8 @@ jobs: jupyterhub_enabled: false flowable_enabled: false wiki_enabled: false - - name: user-minio - display_name: "User Tests (MinIO)" + - name: user-rustfs + display_name: "User Tests (RustFS)" include_admin: false skip_admin: true skip_metadata: false @@ -231,22 +231,22 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: true + rustfs_enabled: true s3compatsigv4_enabled: true weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false flowable_enabled: false wiki_enabled: false - - name: admin-minio - display_name: "Admin Tests (MinIO)" + - name: admin-rustfs + display_name: "Admin Tests (RustFS)" include_admin: true skip_admin: true skip_metadata: true skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: true + rustfs_enabled: true s3compatsigv4_institutional_storage: true weko_enabled: false mibyo_db_enabled: false @@ -261,7 +261,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: true weko_test_mode: direct mibyo_db_enabled: false @@ -276,7 +276,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: true weko_test_mode: workflow mibyo_db_enabled: false @@ -291,7 +291,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: true @@ -305,7 +305,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -320,7 +320,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -335,7 +335,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -351,7 +351,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -366,7 +366,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: true jupyterhub_enabled: false @@ -374,7 +374,7 @@ jobs: wiki_enabled: false name: E2E ${{ matrix.test-group.display_name }} env: - MINIO_ENABLED: ${{ matrix.test-group.minio_enabled == true && 'true' || 'false' }} + RUSTFS_ENABLED: ${{ matrix.test-group.rustfs_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_ENABLED: ${{ matrix.test-group.s3compatsigv4_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_INST_ENABLED: ${{ matrix.test-group.s3compatsigv4_institutional_storage == true && 'true' || 'false' }} WEKO_ENABLED: ${{ matrix.test-group.weko_enabled == true && 'true' || 'false' }} @@ -382,12 +382,12 @@ jobs: FLOWABLE_ENABLED: ${{ matrix.test-group.flowable_enabled == true && 'true' || 'false' }} MIBYO_DB_ENABLED: ${{ matrix.test-group.mibyo_db_enabled == true && 'true' || 'false' }} WIKI_ENABLED: ${{ (matrix.test-group.wiki_enabled == true || github.event.inputs.enable_wiki == 'true') && 'true' || 'false' }} - MINIO_ROOT_USER: e2eadmin - MINIO_ROOT_PASSWORD: e2epassword123 - MINIO_ALIAS: ci-minio - MINIO_ENDPOINT: http://minio:9000 - MINIO_REGION: us-east-1 - MINIO_SERVICE_NAME: MinIO (CI) + RUSTFS_ACCESS_KEY: e2eadmin + RUSTFS_SECRET_KEY: e2epassword123 + RUSTFS_ALIAS: ci-rustfs + RUSTFS_ENDPOINT: http://rustfs:9000 + RUSTFS_REGION: us-east-1 + RUSTFS_SERVICE_NAME: RustFS (CI) steps: - name: Checkout test repository @@ -1033,8 +1033,8 @@ jobs: docker-compose exec -T web python3 /tmp/seed_kaken_es.py /tmp/kaken_seed.json "$NRID_URL" rm "$JSON_FILE" - - name: Initialize MinIO for S3 compatibility tests - if: env.MINIO_ENABLED == 'true' + - name: Initialize RustFS for S3 compatibility tests + if: env.RUSTFS_ENABLED == 'true' working-directory: e2e-tests run: | set -eu @@ -1056,14 +1056,14 @@ jobs: BUCKET_NAME_1="${BUCKET_PREFIX}-a" BUCKET_NAME_2="${BUCKET_PREFIX}-b" - export MINIO_VERIFY_LARGE_UPLOAD=true + export RUSTFS_VERIFY_LARGE_UPLOAD=true - .github/scripts/setup_minio_buckets.sh \ + .github/scripts/setup_rustfs_buckets.sh \ ../RDM-osf.io \ - "${MINIO_ALIAS}" \ - "${MINIO_ENDPOINT}" \ - "${MINIO_ROOT_USER}" \ - "${MINIO_ROOT_PASSWORD}" \ + "${RUSTFS_ALIAS}" \ + "${RUSTFS_ENDPOINT}" \ + "${RUSTFS_ACCESS_KEY}" \ + "${RUSTFS_SECRET_KEY}" \ "${ACCESS_KEY_1}" "${SECRET_KEY_1}" "${BUCKET_NAME_1}" \ "${ACCESS_KEY_2}" "${SECRET_KEY_2}" "${BUCKET_NAME_2}" @@ -1073,12 +1073,12 @@ jobs: echo "S3COMPAT_SECRET_KEY_2=${SECRET_KEY_2}" >> $GITHUB_ENV echo "S3COMPAT_BUCKET_NAME_1=${BUCKET_NAME_1}" >> $GITHUB_ENV echo "S3COMPAT_BUCKET_NAME_2=${BUCKET_NAME_2}" >> $GITHUB_ENV - echo "S3COMPAT_REGION=${MINIO_REGION}" >> $GITHUB_ENV - echo "S3COMPAT_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV - echo "S3COMPAT_SERVICE_NAME=${MINIO_SERVICE_NAME}" >> $GITHUB_ENV + echo "S3COMPAT_REGION=${RUSTFS_REGION}" >> $GITHUB_ENV + echo "S3COMPAT_ENDPOINT=${RUSTFS_ENDPOINT}" >> $GITHUB_ENV + echo "S3COMPAT_SERVICE_NAME=${RUSTFS_SERVICE_NAME}" >> $GITHUB_ENV - - name: Initialize MinIO for S3CompatSigV4 tests - if: env.MINIO_ENABLED == 'true' && (env.S3COMPATSIGV4_ENABLED == 'true' || env.S3COMPATSIGV4_INST_ENABLED == 'true') + - name: Initialize RustFS for S3CompatSigV4 tests + if: env.RUSTFS_ENABLED == 'true' && (env.S3COMPATSIGV4_ENABLED == 'true' || env.S3COMPATSIGV4_INST_ENABLED == 'true') working-directory: e2e-tests run: | set -eu @@ -1102,12 +1102,12 @@ jobs: BUCKET_NAME_1="${BUCKET_PREFIX}-a" BUCKET_NAME_2="${BUCKET_PREFIX}-b" - .github/scripts/setup_minio_buckets.sh \ + .github/scripts/setup_rustfs_buckets.sh \ ../RDM-osf.io \ - "${MINIO_ALIAS}" \ - "${MINIO_ENDPOINT}" \ - "${MINIO_ROOT_USER}" \ - "${MINIO_ROOT_PASSWORD}" \ + "${RUSTFS_ALIAS}" \ + "${RUSTFS_ENDPOINT}" \ + "${RUSTFS_ACCESS_KEY}" \ + "${RUSTFS_SECRET_KEY}" \ "${ACCESS_KEY_1}" "${SECRET_KEY_1}" "${BUCKET_NAME_1}" \ "${ACCESS_KEY_2}" "${SECRET_KEY_2}" "${BUCKET_NAME_2}" @@ -1117,44 +1117,32 @@ jobs: echo "S3COMPATSIGV4_SECRET_KEY_2=${SECRET_KEY_2}" >> $GITHUB_ENV echo "S3COMPATSIGV4_BUCKET_NAME_1=${BUCKET_NAME_1}" >> $GITHUB_ENV echo "S3COMPATSIGV4_BUCKET_NAME_2=${BUCKET_NAME_2}" >> $GITHUB_ENV - echo "S3COMPATSIGV4_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV - echo "S3COMPATSIGV4_SERVICE_NAME=${MINIO_SERVICE_NAME}" >> $GITHUB_ENV + echo "S3COMPATSIGV4_ENDPOINT=${RUSTFS_ENDPOINT}" >> $GITHUB_ENV + echo "S3COMPATSIGV4_SERVICE_NAME=${RUSTFS_SERVICE_NAME}" >> $GITHUB_ENV fi if [ "${S3COMPATSIGV4_INST_ENABLED}" = "true" ]; then INST_ACCESS_KEY="ci$(key_part)" INST_SECRET_KEY="$(secret_part)" INST_BUCKET="${BUCKET_PREFIX}-inst" - # Second bucket is required by setup_minio_buckets.sh but not used for inst storage + # Second bucket is required by setup_rustfs_buckets.sh but not used for inst storage INST_BUCKET_DUMMY="${BUCKET_PREFIX}-inst2" - .github/scripts/setup_minio_buckets.sh \ + .github/scripts/setup_rustfs_buckets.sh \ ../RDM-osf.io \ - "${MINIO_ALIAS}" \ - "${MINIO_ENDPOINT}" \ - "${MINIO_ROOT_USER}" \ - "${MINIO_ROOT_PASSWORD}" \ + "${RUSTFS_ALIAS}" \ + "${RUSTFS_ENDPOINT}" \ + "${RUSTFS_ACCESS_KEY}" \ + "${RUSTFS_SECRET_KEY}" \ "${INST_ACCESS_KEY}" "${INST_SECRET_KEY}" "${INST_BUCKET}" \ "${INST_ACCESS_KEY}" "${INST_SECRET_KEY}" "${INST_BUCKET_DUMMY}" echo "S3COMPATSIGV4_INST_ACCESS_KEY=${INST_ACCESS_KEY}" >> $GITHUB_ENV echo "S3COMPATSIGV4_INST_SECRET_KEY=${INST_SECRET_KEY}" >> $GITHUB_ENV echo "S3COMPATSIGV4_INST_BUCKET=${INST_BUCKET}" >> $GITHUB_ENV - echo "S3COMPATSIGV4_INST_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV + echo "S3COMPATSIGV4_INST_ENDPOINT=${RUSTFS_ENDPOINT}" >> $GITHUB_ENV fi - - name: Start MinIO trace - if: env.MINIO_ENABLED == 'true' - working-directory: RDM-osf.io - run: | - set -eu - docker-compose run -d --name minio-trace --entrypoint /bin/sh minio-mc -c " - set -euo pipefail - mc alias set ${MINIO_ALIAS} ${MINIO_ENDPOINT} ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} - mc admin trace --json --verbose --errors --path '${S3COMPAT_BUCKET_NAME_1}/*' --path '${S3COMPAT_BUCKET_NAME_2}/*' ${MINIO_ALIAS} - " - - - name: Test endpoint accessibility working-directory: RDM-osf.io run: | @@ -1374,8 +1362,8 @@ jobs: EOF args=(ci.config.yaml base_ci.config.yaml) - if [ "${MINIO_ENABLED}" = "true" ]; then - args+=(--minio) + if [ "${RUSTFS_ENABLED}" = "true" ]; then + args+=(--rustfs) fi if [ "${JUPYTERHUB_ENABLED}" = "true" ]; then args+=(--jupyterhub) @@ -1502,24 +1490,6 @@ jobs: path: e2e-tests/result/ retention-days: 7 - - name: Collect MinIO trace - if: always() && env.MINIO_ENABLED == 'true' - working-directory: RDM-osf.io - run: | - set -eu - if docker ps -a --format '{{.Names}}' | grep -q '^minio-trace$'; then - docker logs minio-trace > ../e2e-tests/minio-trace.log || true - docker rm -f minio-trace || true - fi - - - name: Upload MinIO trace - if: always() && env.MINIO_ENABLED == 'true' - uses: actions/upload-artifact@v4 - with: - name: minio-trace-${{ matrix.test-group.name }} - path: e2e-tests/minio-trace.log - if-no-files-found: ignore - - name: Collect WEKO logs if: always() && (env.WEKO_ENABLED == 'true' || env.MIBYO_DB_ENABLED == 'true') working-directory: weko From fac7ea88968c614769b1daed1b098be6e64b7e95 Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:31:52 +0900 Subject: [PATCH 2/3] Collect RustFS container logs as an artifact --- .github/scripts/setup_rustfs.sh | 1 + .github/workflows/e2e-test.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/.github/scripts/setup_rustfs.sh b/.github/scripts/setup_rustfs.sh index 503acf0..e20d1c7 100755 --- a/.github/scripts/setup_rustfs.sh +++ b/.github/scripts/setup_rustfs.sh @@ -39,6 +39,7 @@ RUSTFS_DOCKER_SNIPPET=$(cat < ../e2e-tests/rustfs-logs.txt 2>&1 + + - name: Upload RustFS logs + if: always() && env.RUSTFS_ENABLED == 'true' + uses: actions/upload-artifact@v4 + with: + name: rustfs-logs-${{ matrix.test-group.name }} + path: e2e-tests/rustfs-logs.txt + retention-days: 7 + - name: Collect WEKO logs if: always() && (env.WEKO_ENABLED == 'true' || env.MIBYO_DB_ENABLED == 'true') working-directory: weko From 7381f94e0f29d57319dc90307e12ab263d5ea4de Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:52:13 +0900 Subject: [PATCH 3/3] Disable SigV2 s3compat tests on RustFS --- .github/scripts/generate_ci_config.sh | 36 ++----------------- .github/scripts/setup_rustfs.sh | 52 ++------------------------- .github/workflows/e2e-test.yml | 50 ++------------------------ 3 files changed, 7 insertions(+), 131 deletions(-) diff --git a/.github/scripts/generate_ci_config.sh b/.github/scripts/generate_ci_config.sh index d643f3c..aaa0752 100755 --- a/.github/scripts/generate_ci_config.sh +++ b/.github/scripts/generate_ci_config.sh @@ -3,7 +3,7 @@ set -xeuo pipefail if [[ $# -lt 2 ]]; then cat >&2 <<'USAGE' -Usage: generate_ci_config.sh [--rustfs] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] +Usage: generate_ci_config.sh [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] USAGE exit 1 fi @@ -11,7 +11,6 @@ fi OUTPUT=$1 BASE_CONFIG=$2 shift 2 -RUSTFS=false JUPYTERHUB=false WEKO=false FLOWABLE=false @@ -21,9 +20,6 @@ WIKI=false for arg in "$@"; do case "$arg" in - --rustfs) - RUSTFS=true - ;; --jupyterhub) JUPYTERHUB=true ;; @@ -52,38 +48,10 @@ done cp "${BASE_CONFIG}" "${OUTPUT}" -if [[ "${RUSTFS}" == "true" ]]; then - if [[ -z "${S3COMPAT_ACCESS_KEY_1:-}" || -z "${S3COMPAT_SECRET_KEY_1:-}" ]]; then - echo "S3 compat credentials are not set" >&2 - exit 1 - fi - - cat >> "${OUTPUT}" <> "${OUTPUT}" <<'EOF' +cat >> "${OUTPUT}" <<'EOF' storages_s3: [] EOF -fi if [[ "${JUPYTERHUB}" == "true" ]]; then if [[ -z "${TLJH_URL:-}" || -z "${TLJH_USERNAME:-}" || -z "${TLJH_PASSWORD:-}" ]]; then diff --git a/.github/scripts/setup_rustfs.sh b/.github/scripts/setup_rustfs.sh index e20d1c7..a875494 100755 --- a/.github/scripts/setup_rustfs.sh +++ b/.github/scripts/setup_rustfs.sh @@ -4,7 +4,7 @@ set -xeuo pipefail # Usage: # ./setup_rustfs.sh apply # -# Generates docker-compose override entries and s3compat settings +# Generates docker-compose override entries and s3compatsigv4 settings # required for RustFS usage within the E2E environment. COMMAND=${1:-} @@ -61,10 +61,10 @@ if ! grep -q '^ rustfs:' "${compose_override}"; then printf '\n%s\n' "${RUSTFS_DOCKER_SNIPPET}" >> "${compose_override}" fi -settings_json="${RDM_ROOT}/addons/s3compat/static/settings.json" +settings_json="${RDM_ROOT}/addons/s3compatsigv4/static/settings.json" if [[ ! -f "${settings_json}" ]]; then - echo "s3compat settings.json not found: ${settings_json}" >&2 + echo "s3compatsigv4 settings.json not found: ${settings_json}" >&2 exit 1 fi @@ -106,50 +106,4 @@ PY mv "${tmp_json}" "${settings_json}" -# Also register RustFS service for s3compatsigv4 addon if available -sigv4_settings_json="${RDM_ROOT}/addons/s3compatsigv4/static/settings.json" - -if [[ -f "${sigv4_settings_json}" ]]; then - tmp_json_sigv4=$(mktemp) - - python - "$sigv4_settings_json" "$tmp_json_sigv4" <<'PY' -import json -import sys - -src, dst = sys.argv[1:3] -with open(src) as f: - data = json.load(f) - -service_name = "RustFS (CI)" -host = "rustfs:9000" - -available_services = data.setdefault("availableServices", []) - -if not any(s.get("name") == service_name for s in available_services): - available_services.append({ - "name": service_name, - "host": host, - "bucketLocations": { - "us-east-1": { - "name": "us-east-1", - "host": host, - }, - "": { - "name": "us-east-1", - }, - }, - "serverSideEncryption": False, - }) - -with open(dst, "w") as f: - json.dump(data, f, ensure_ascii=False, indent=4) - f.write("\n") -PY - - mv "${tmp_json_sigv4}" "${sigv4_settings_json}" - echo "RustFS configuration applied for s3compatsigv4" -else - echo "WARNING: s3compatsigv4 settings.json not found: ${sigv4_settings_json} (skipping)" -fi - echo "RustFS configuration applied" diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 28dbecb..496fe1f 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -386,7 +386,6 @@ jobs: RUSTFS_SECRET_KEY: e2epassword123 RUSTFS_ALIAS: ci-rustfs RUSTFS_ENDPOINT: http://rustfs:9000 - RUSTFS_REGION: us-east-1 RUSTFS_SERVICE_NAME: RustFS (CI) steps: @@ -1033,50 +1032,6 @@ jobs: docker-compose exec -T web python3 /tmp/seed_kaken_es.py /tmp/kaken_seed.json "$NRID_URL" rm "$JSON_FILE" - - name: Initialize RustFS for S3 compatibility tests - if: env.RUSTFS_ENABLED == 'true' - working-directory: e2e-tests - run: | - set -eu - - key_part() { - LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 18 - } - - secret_part() { - LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 40 - } - - ACCESS_KEY_1="ci$(key_part)" - SECRET_KEY_1="$(secret_part)" - ACCESS_KEY_2="ci$(key_part)" - SECRET_KEY_2="$(secret_part)" - - BUCKET_PREFIX="ci-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}" - BUCKET_NAME_1="${BUCKET_PREFIX}-a" - BUCKET_NAME_2="${BUCKET_PREFIX}-b" - - export RUSTFS_VERIFY_LARGE_UPLOAD=true - - .github/scripts/setup_rustfs_buckets.sh \ - ../RDM-osf.io \ - "${RUSTFS_ALIAS}" \ - "${RUSTFS_ENDPOINT}" \ - "${RUSTFS_ACCESS_KEY}" \ - "${RUSTFS_SECRET_KEY}" \ - "${ACCESS_KEY_1}" "${SECRET_KEY_1}" "${BUCKET_NAME_1}" \ - "${ACCESS_KEY_2}" "${SECRET_KEY_2}" "${BUCKET_NAME_2}" - - echo "S3COMPAT_ACCESS_KEY_1=${ACCESS_KEY_1}" >> $GITHUB_ENV - echo "S3COMPAT_SECRET_KEY_1=${SECRET_KEY_1}" >> $GITHUB_ENV - echo "S3COMPAT_ACCESS_KEY_2=${ACCESS_KEY_2}" >> $GITHUB_ENV - echo "S3COMPAT_SECRET_KEY_2=${SECRET_KEY_2}" >> $GITHUB_ENV - echo "S3COMPAT_BUCKET_NAME_1=${BUCKET_NAME_1}" >> $GITHUB_ENV - echo "S3COMPAT_BUCKET_NAME_2=${BUCKET_NAME_2}" >> $GITHUB_ENV - echo "S3COMPAT_REGION=${RUSTFS_REGION}" >> $GITHUB_ENV - echo "S3COMPAT_ENDPOINT=${RUSTFS_ENDPOINT}" >> $GITHUB_ENV - echo "S3COMPAT_SERVICE_NAME=${RUSTFS_SERVICE_NAME}" >> $GITHUB_ENV - - name: Initialize RustFS for S3CompatSigV4 tests if: env.RUSTFS_ENABLED == 'true' && (env.S3COMPATSIGV4_ENABLED == 'true' || env.S3COMPATSIGV4_INST_ENABLED == 'true') working-directory: e2e-tests @@ -1093,6 +1048,8 @@ jobs: BUCKET_PREFIX="ci-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-sigv4" + export RUSTFS_VERIFY_LARGE_UPLOAD=true + if [ "${S3COMPATSIGV4_ENABLED}" = "true" ]; then ACCESS_KEY_1="ci$(key_part)" SECRET_KEY_1="$(secret_part)" @@ -1362,9 +1319,6 @@ jobs: EOF args=(ci.config.yaml base_ci.config.yaml) - if [ "${RUSTFS_ENABLED}" = "true" ]; then - args+=(--rustfs) - fi if [ "${JUPYTERHUB_ENABLED}" = "true" ]; then args+=(--jupyterhub) fi