diff --git a/.github/scripts/generate_ci_config.sh b/.github/scripts/generate_ci_config.sh index 86d960c..aaa0752 100755 --- a/.github/scripts/generate_ci_config.sh +++ b/.github/scripts/generate_ci_config.sh @@ -3,7 +3,7 @@ set -xeuo pipefail if [[ $# -lt 2 ]]; then cat >&2 <<'USAGE' -Usage: generate_ci_config.sh [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] +Usage: generate_ci_config.sh [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] USAGE exit 1 fi @@ -11,7 +11,6 @@ fi OUTPUT=$1 BASE_CONFIG=$2 shift 2 -MINIO=false JUPYTERHUB=false WEKO=false FLOWABLE=false @@ -21,9 +20,6 @@ WIKI=false for arg in "$@"; do case "$arg" in - --minio) - MINIO=true - ;; --jupyterhub) JUPYTERHUB=true ;; @@ -52,38 +48,10 @@ done cp "${BASE_CONFIG}" "${OUTPUT}" -if [[ "${MINIO}" == "true" ]]; then - if [[ -z "${S3COMPAT_ACCESS_KEY_1:-}" || -z "${S3COMPAT_SECRET_KEY_1:-}" ]]; then - echo "S3 compat credentials are not set" >&2 - exit 1 - fi - - cat >> "${OUTPUT}" <> "${OUTPUT}" <<'EOF' +cat >> "${OUTPUT}" <<'EOF' storages_s3: [] EOF -fi if [[ "${JUPYTERHUB}" == "true" ]]; then if [[ -z "${TLJH_URL:-}" || -z "${TLJH_USERNAME:-}" || -z "${TLJH_PASSWORD:-}" ]]; then diff --git a/.github/scripts/setup_minio.sh b/.github/scripts/setup_minio.sh deleted file mode 100755 index 685cf82..0000000 --- a/.github/scripts/setup_minio.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash -set -xeuo pipefail - -# Usage: -# ./setup_minio.sh apply -# -# Generates docker-compose override entries and s3compat settings -# required for MinIO usage within the E2E environment. - -COMMAND=${1:-} -RDM_ROOT=${2:-} - -if [[ -z "${COMMAND}" || -z "${RDM_ROOT}" ]]; then - echo "Usage: $0 apply " >&2 - exit 1 -fi - -case "${COMMAND}" in - apply) - ;; - *) - echo "Unknown command: ${COMMAND}" >&2 - exit 1 - ;; -esac - -if [[ ! -d "${RDM_ROOT}" ]]; then - echo "RDM root directory not found: ${RDM_ROOT}" >&2 - exit 1 -fi - -MINIO_IMAGE_DEFAULT=${MINIO_IMAGE:-minio/minio:latest} -MINIO_MC_IMAGE_DEFAULT=${MINIO_MC_IMAGE:-minio/mc:latest} - -MINIO_DOCKER_SNIPPET=$(cat < "${compose_override}" -fi - -if ! grep -q '^ minio:' "${compose_override}"; then - printf '\n%s\n' "${MINIO_DOCKER_SNIPPET}" >> "${compose_override}" -fi - -settings_json="${RDM_ROOT}/addons/s3compat/static/settings.json" - -if [[ ! -f "${settings_json}" ]]; then - echo "s3compat settings.json not found: ${settings_json}" >&2 - exit 1 -fi - -tmp_json=$(mktemp) - -python - "$settings_json" "$tmp_json" <<'PY' -import json -import sys - -src, dst = sys.argv[1:3] -with open(src) as f: - data = json.load(f) - -service_name = "MinIO (CI)" -host = "minio:9000" - -available_services = data.setdefault("availableServices", []) - -if not any(s.get("name") == service_name for s in available_services): - available_services.append({ - "name": service_name, - "host": host, - "bucketLocations": { - "us-east-1": { - "name": "us-east-1", - "host": host, - }, - "": { - "name": "us-east-1", - }, - }, - "serverSideEncryption": False, - }) - -with open(dst, "w") as f: - json.dump(data, f, ensure_ascii=False, indent=4) - f.write("\n") -PY - -mv "${tmp_json}" "${settings_json}" - -# Also register MinIO service for s3compatsigv4 addon if available -sigv4_settings_json="${RDM_ROOT}/addons/s3compatsigv4/static/settings.json" - -if [[ -f "${sigv4_settings_json}" ]]; then - tmp_json_sigv4=$(mktemp) - - python - "$sigv4_settings_json" "$tmp_json_sigv4" <<'PY' -import json -import sys - -src, dst = sys.argv[1:3] -with open(src) as f: - data = json.load(f) - -service_name = "MinIO (CI)" -host = "minio:9000" - -available_services = data.setdefault("availableServices", []) - -if not any(s.get("name") == service_name for s in available_services): - available_services.append({ - "name": service_name, - "host": host, - "bucketLocations": { - "us-east-1": { - "name": "us-east-1", - "host": host, - }, - "": { - "name": "us-east-1", - }, - }, - "serverSideEncryption": False, - }) - -with open(dst, "w") as f: - json.dump(data, f, ensure_ascii=False, indent=4) - f.write("\n") -PY - - mv "${tmp_json_sigv4}" "${sigv4_settings_json}" - echo "MinIO configuration applied for s3compatsigv4" -else - echo "WARNING: s3compatsigv4 settings.json not found: ${sigv4_settings_json} (skipping)" -fi - -echo "MinIO configuration applied" diff --git a/.github/scripts/setup_minio_buckets.sh b/.github/scripts/setup_minio_buckets.sh deleted file mode 100755 index 479d0b8..0000000 --- a/.github/scripts/setup_minio_buckets.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -xeuo pipefail - -# Usage: -# ./setup_minio_buckets.sh \ -# - -if [[ $# -ne 11 ]]; then - cat >&2 <<'EOF' -Usage: setup_minio_buckets.sh \ - -EOF - exit 1 -fi - -RDM_ROOT=$1 -ALIAS=$2 -ENDPOINT=$3 -ROOT_USER=$4 -ROOT_PASSWORD=$5 -ACCESS_KEY_1=$6 -SECRET_KEY_1=$7 -BUCKET_NAME_1=$8 -ACCESS_KEY_2=$9 -SECRET_KEY_2=${10} -BUCKET_NAME_2=${11} - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -MINIO_VERIFY_LARGE_UPLOAD=${MINIO_VERIFY_LARGE_UPLOAD:-false} -MINIO_VERIFY_LARGE_UPLOAD_SIZE_MB=${MINIO_VERIFY_LARGE_UPLOAD_SIZE_MB:-130} -MINIO_VERIFY_LARGE_UPLOAD_KEY=${MINIO_VERIFY_LARGE_UPLOAD_KEY:-diagnostics/minio-large-upload.bin} - -if [[ ! -d "${RDM_ROOT}" ]]; then - echo "RDM root directory not found: ${RDM_ROOT}" >&2 - exit 1 -fi - -pushd "${RDM_ROOT}" >/dev/null -source "${SCRIPT_DIR}/lib/wait_for_service.sh" - -SERVICE_NAME="MinIO" -CHECK_COMMAND="docker-compose exec -T minio /bin/sh -c 'curl -fs http://localhost:9000/minio/health/ready'" -TIMEOUT=180 -INTERVAL=5 -wait_for_service - -docker-compose run --rm --entrypoint /bin/sh minio-mc <&2 - dd if=/dev/zero of=/tmp/minio-large-upload-test.bin bs=1M count=${MINIO_VERIFY_LARGE_UPLOAD_SIZE_MB} - mc cp /tmp/minio-large-upload-test.bin "${ALIAS}/${BUCKET_NAME_1}/${MINIO_VERIFY_LARGE_UPLOAD_KEY}" - mc stat "${ALIAS}/${BUCKET_NAME_1}/${MINIO_VERIFY_LARGE_UPLOAD_KEY}" - rm -f /tmp/minio-large-upload-test.bin -fi -EOF - -popd >/dev/null - -echo "MinIO buckets and users have been initialized" diff --git a/.github/scripts/setup_rdm.sh b/.github/scripts/setup_rdm.sh index 2111510..7975c91 100644 --- a/.github/scripts/setup_rdm.sh +++ b/.github/scripts/setup_rdm.sh @@ -85,8 +85,8 @@ start_rdm_services() { SERVICES="kaken_elasticsearch $SERVICES" - if [ "${MINIO_ENABLED:-false}" = "true" ]; then - SERVICES="$SERVICES minio" + if [ "${RUSTFS_ENABLED:-false}" = "true" ]; then + SERVICES="$SERVICES rustfs" fi echo "Starting services: $SERVICES" @@ -336,10 +336,10 @@ EOL echo "" >> .docker-compose.wb.env echo "SERVER_CONFIG_DOMAIN=http://192.168.168.167:7777" >> .docker-compose.wb.env - if [ "${MINIO_ENABLED:-false}" = "true" ]; then + if [ "${RUSTFS_ENABLED:-false}" = "true" ]; then local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - bash "${script_dir}/setup_minio.sh" apply "${PWD}" + bash "${script_dir}/setup_rustfs.sh" apply "${PWD}" fi if [ "${WEKO_ENABLED:-false}" = "true" ] || [ "${MIBYO_DB_ENABLED:-false}" = "true" ]; then diff --git a/.github/scripts/setup_rustfs.sh b/.github/scripts/setup_rustfs.sh new file mode 100755 index 0000000..a875494 --- /dev/null +++ b/.github/scripts/setup_rustfs.sh @@ -0,0 +1,109 @@ +#!/bin/bash +set -xeuo pipefail + +# Usage: +# ./setup_rustfs.sh apply +# +# Generates docker-compose override entries and s3compatsigv4 settings +# required for RustFS usage within the E2E environment. + +COMMAND=${1:-} +RDM_ROOT=${2:-} + +if [[ -z "${COMMAND}" || -z "${RDM_ROOT}" ]]; then + echo "Usage: $0 apply " >&2 + exit 1 +fi + +case "${COMMAND}" in + apply) + ;; + *) + echo "Unknown command: ${COMMAND}" >&2 + exit 1 + ;; +esac + +if [[ ! -d "${RDM_ROOT}" ]]; then + echo "RDM root directory not found: ${RDM_ROOT}" >&2 + exit 1 +fi + +RUSTFS_IMAGE_DEFAULT=${RUSTFS_IMAGE:-rustfs/rustfs:latest} +RUSTFS_RC_IMAGE_DEFAULT=${RUSTFS_RC_IMAGE:-rustfs/rc:latest} + +RUSTFS_DOCKER_SNIPPET=$(cat < "${compose_override}" +fi + +if ! grep -q '^ rustfs:' "${compose_override}"; then + printf '\n%s\n' "${RUSTFS_DOCKER_SNIPPET}" >> "${compose_override}" +fi + +settings_json="${RDM_ROOT}/addons/s3compatsigv4/static/settings.json" + +if [[ ! -f "${settings_json}" ]]; then + echo "s3compatsigv4 settings.json not found: ${settings_json}" >&2 + exit 1 +fi + +tmp_json=$(mktemp) + +python - "$settings_json" "$tmp_json" <<'PY' +import json +import sys + +src, dst = sys.argv[1:3] +with open(src) as f: + data = json.load(f) + +service_name = "RustFS (CI)" +host = "rustfs:9000" + +available_services = data.setdefault("availableServices", []) + +if not any(s.get("name") == service_name for s in available_services): + available_services.append({ + "name": service_name, + "host": host, + "bucketLocations": { + "us-east-1": { + "name": "us-east-1", + "host": host, + }, + "": { + "name": "us-east-1", + }, + }, + "serverSideEncryption": False, + }) + +with open(dst, "w") as f: + json.dump(data, f, ensure_ascii=False, indent=4) + f.write("\n") +PY + +mv "${tmp_json}" "${settings_json}" + +echo "RustFS configuration applied" diff --git a/.github/scripts/setup_rustfs_buckets.sh b/.github/scripts/setup_rustfs_buckets.sh new file mode 100755 index 0000000..f7f65ec --- /dev/null +++ b/.github/scripts/setup_rustfs_buckets.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -xeuo pipefail + +# Usage: +# ./setup_rustfs_buckets.sh \ +# + +if [[ $# -ne 11 ]]; then + cat >&2 <<'EOF' +Usage: setup_rustfs_buckets.sh \ + +EOF + exit 1 +fi + +RDM_ROOT=$1 +ALIAS=$2 +ENDPOINT=$3 +ROOT_USER=$4 +ROOT_PASSWORD=$5 +ACCESS_KEY_1=$6 +SECRET_KEY_1=$7 +BUCKET_NAME_1=$8 +ACCESS_KEY_2=$9 +SECRET_KEY_2=${10} +BUCKET_NAME_2=${11} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RUSTFS_VERIFY_LARGE_UPLOAD=${RUSTFS_VERIFY_LARGE_UPLOAD:-false} +RUSTFS_VERIFY_LARGE_UPLOAD_SIZE_MB=${RUSTFS_VERIFY_LARGE_UPLOAD_SIZE_MB:-130} +RUSTFS_VERIFY_LARGE_UPLOAD_KEY=${RUSTFS_VERIFY_LARGE_UPLOAD_KEY:-diagnostics/rustfs-large-upload.bin} + +if [[ ! -d "${RDM_ROOT}" ]]; then + echo "RDM root directory not found: ${RDM_ROOT}" >&2 + exit 1 +fi + +pushd "${RDM_ROOT}" >/dev/null +source "${SCRIPT_DIR}/lib/wait_for_service.sh" + +SERVICE_NAME="RustFS" +CHECK_COMMAND="docker-compose exec -T rustfs /bin/sh -c 'curl -fs http://localhost:9000/health/ready'" +TIMEOUT=180 +INTERVAL=5 +wait_for_service + +docker-compose run --rm --entrypoint /bin/sh rustfs-rc <&2 + dd if=/dev/zero of=/tmp/rustfs-large-upload-test.bin bs=1M count=${RUSTFS_VERIFY_LARGE_UPLOAD_SIZE_MB} + rc object copy /tmp/rustfs-large-upload-test.bin "${ALIAS}/${BUCKET_NAME_1}/${RUSTFS_VERIFY_LARGE_UPLOAD_KEY}" + rc object stat "${ALIAS}/${BUCKET_NAME_1}/${RUSTFS_VERIFY_LARGE_UPLOAD_KEY}" + rm -f /tmp/rustfs-large-upload-test.bin +fi +EOF + +popd >/dev/null + +echo "RustFS buckets and users have been initialized" diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 8897a8a..496fe1f 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -118,7 +118,7 @@ jobs: skip_admin: true skip_metadata: false skip_autofill: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -133,7 +133,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -146,7 +146,7 @@ jobs: skip_metadata: true skip_default_storage: true skip_login: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -189,7 +189,7 @@ jobs: テスト手順-管理者機能-アナウンス.ipynb テスト手順-管理者機能-証跡管理.ipynb テスト手順-管理者機能-NIIストレージのクォータ.ipynb - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -211,7 +211,7 @@ jobs: exclude_notebooks: | テスト手順-WEKOアドオン-メタデータ送信-書誌情報.ipynb テスト手順-WEKOアドオン-メタデータ送信-書誌情報+根拠データ.ipynb - minio_enabled: false + rustfs_enabled: false weko_enabled: true mibyo_db_enabled: false weko_test_mode: direct @@ -222,8 +222,8 @@ jobs: jupyterhub_enabled: false flowable_enabled: false wiki_enabled: false - - name: user-minio - display_name: "User Tests (MinIO)" + - name: user-rustfs + display_name: "User Tests (RustFS)" include_admin: false skip_admin: true skip_metadata: false @@ -231,22 +231,22 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: true + rustfs_enabled: true s3compatsigv4_enabled: true weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false flowable_enabled: false wiki_enabled: false - - name: admin-minio - display_name: "Admin Tests (MinIO)" + - name: admin-rustfs + display_name: "Admin Tests (RustFS)" include_admin: true skip_admin: true skip_metadata: true skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: true + rustfs_enabled: true s3compatsigv4_institutional_storage: true weko_enabled: false mibyo_db_enabled: false @@ -261,7 +261,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: true weko_test_mode: direct mibyo_db_enabled: false @@ -276,7 +276,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: true weko_test_mode: workflow mibyo_db_enabled: false @@ -291,7 +291,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: true @@ -305,7 +305,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -320,7 +320,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -335,7 +335,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -351,7 +351,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: false jupyterhub_enabled: false @@ -366,7 +366,7 @@ jobs: skip_default_storage: true skip_login: true skip_130mb_upload: true - minio_enabled: false + rustfs_enabled: false weko_enabled: false mibyo_db_enabled: true jupyterhub_enabled: false @@ -374,7 +374,7 @@ jobs: wiki_enabled: false name: E2E ${{ matrix.test-group.display_name }} env: - MINIO_ENABLED: ${{ matrix.test-group.minio_enabled == true && 'true' || 'false' }} + RUSTFS_ENABLED: ${{ matrix.test-group.rustfs_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_ENABLED: ${{ matrix.test-group.s3compatsigv4_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_INST_ENABLED: ${{ matrix.test-group.s3compatsigv4_institutional_storage == true && 'true' || 'false' }} WEKO_ENABLED: ${{ matrix.test-group.weko_enabled == true && 'true' || 'false' }} @@ -382,12 +382,11 @@ jobs: FLOWABLE_ENABLED: ${{ matrix.test-group.flowable_enabled == true && 'true' || 'false' }} MIBYO_DB_ENABLED: ${{ matrix.test-group.mibyo_db_enabled == true && 'true' || 'false' }} WIKI_ENABLED: ${{ (matrix.test-group.wiki_enabled == true || github.event.inputs.enable_wiki == 'true') && 'true' || 'false' }} - MINIO_ROOT_USER: e2eadmin - MINIO_ROOT_PASSWORD: e2epassword123 - MINIO_ALIAS: ci-minio - MINIO_ENDPOINT: http://minio:9000 - MINIO_REGION: us-east-1 - MINIO_SERVICE_NAME: MinIO (CI) + RUSTFS_ACCESS_KEY: e2eadmin + RUSTFS_SECRET_KEY: e2epassword123 + RUSTFS_ALIAS: ci-rustfs + RUSTFS_ENDPOINT: http://rustfs:9000 + RUSTFS_SERVICE_NAME: RustFS (CI) steps: - name: Checkout test repository @@ -1033,52 +1032,8 @@ jobs: docker-compose exec -T web python3 /tmp/seed_kaken_es.py /tmp/kaken_seed.json "$NRID_URL" rm "$JSON_FILE" - - name: Initialize MinIO for S3 compatibility tests - if: env.MINIO_ENABLED == 'true' - working-directory: e2e-tests - run: | - set -eu - - key_part() { - LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 18 - } - - secret_part() { - LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 40 - } - - ACCESS_KEY_1="ci$(key_part)" - SECRET_KEY_1="$(secret_part)" - ACCESS_KEY_2="ci$(key_part)" - SECRET_KEY_2="$(secret_part)" - - BUCKET_PREFIX="ci-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}" - BUCKET_NAME_1="${BUCKET_PREFIX}-a" - BUCKET_NAME_2="${BUCKET_PREFIX}-b" - - export MINIO_VERIFY_LARGE_UPLOAD=true - - .github/scripts/setup_minio_buckets.sh \ - ../RDM-osf.io \ - "${MINIO_ALIAS}" \ - "${MINIO_ENDPOINT}" \ - "${MINIO_ROOT_USER}" \ - "${MINIO_ROOT_PASSWORD}" \ - "${ACCESS_KEY_1}" "${SECRET_KEY_1}" "${BUCKET_NAME_1}" \ - "${ACCESS_KEY_2}" "${SECRET_KEY_2}" "${BUCKET_NAME_2}" - - echo "S3COMPAT_ACCESS_KEY_1=${ACCESS_KEY_1}" >> $GITHUB_ENV - echo "S3COMPAT_SECRET_KEY_1=${SECRET_KEY_1}" >> $GITHUB_ENV - echo "S3COMPAT_ACCESS_KEY_2=${ACCESS_KEY_2}" >> $GITHUB_ENV - echo "S3COMPAT_SECRET_KEY_2=${SECRET_KEY_2}" >> $GITHUB_ENV - echo "S3COMPAT_BUCKET_NAME_1=${BUCKET_NAME_1}" >> $GITHUB_ENV - echo "S3COMPAT_BUCKET_NAME_2=${BUCKET_NAME_2}" >> $GITHUB_ENV - echo "S3COMPAT_REGION=${MINIO_REGION}" >> $GITHUB_ENV - echo "S3COMPAT_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV - echo "S3COMPAT_SERVICE_NAME=${MINIO_SERVICE_NAME}" >> $GITHUB_ENV - - - name: Initialize MinIO for S3CompatSigV4 tests - if: env.MINIO_ENABLED == 'true' && (env.S3COMPATSIGV4_ENABLED == 'true' || env.S3COMPATSIGV4_INST_ENABLED == 'true') + - name: Initialize RustFS for S3CompatSigV4 tests + if: env.RUSTFS_ENABLED == 'true' && (env.S3COMPATSIGV4_ENABLED == 'true' || env.S3COMPATSIGV4_INST_ENABLED == 'true') working-directory: e2e-tests run: | set -eu @@ -1093,6 +1048,8 @@ jobs: BUCKET_PREFIX="ci-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-sigv4" + export RUSTFS_VERIFY_LARGE_UPLOAD=true + if [ "${S3COMPATSIGV4_ENABLED}" = "true" ]; then ACCESS_KEY_1="ci$(key_part)" SECRET_KEY_1="$(secret_part)" @@ -1102,12 +1059,12 @@ jobs: BUCKET_NAME_1="${BUCKET_PREFIX}-a" BUCKET_NAME_2="${BUCKET_PREFIX}-b" - .github/scripts/setup_minio_buckets.sh \ + .github/scripts/setup_rustfs_buckets.sh \ ../RDM-osf.io \ - "${MINIO_ALIAS}" \ - "${MINIO_ENDPOINT}" \ - "${MINIO_ROOT_USER}" \ - "${MINIO_ROOT_PASSWORD}" \ + "${RUSTFS_ALIAS}" \ + "${RUSTFS_ENDPOINT}" \ + "${RUSTFS_ACCESS_KEY}" \ + "${RUSTFS_SECRET_KEY}" \ "${ACCESS_KEY_1}" "${SECRET_KEY_1}" "${BUCKET_NAME_1}" \ "${ACCESS_KEY_2}" "${SECRET_KEY_2}" "${BUCKET_NAME_2}" @@ -1117,44 +1074,32 @@ jobs: echo "S3COMPATSIGV4_SECRET_KEY_2=${SECRET_KEY_2}" >> $GITHUB_ENV echo "S3COMPATSIGV4_BUCKET_NAME_1=${BUCKET_NAME_1}" >> $GITHUB_ENV echo "S3COMPATSIGV4_BUCKET_NAME_2=${BUCKET_NAME_2}" >> $GITHUB_ENV - echo "S3COMPATSIGV4_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV - echo "S3COMPATSIGV4_SERVICE_NAME=${MINIO_SERVICE_NAME}" >> $GITHUB_ENV + echo "S3COMPATSIGV4_ENDPOINT=${RUSTFS_ENDPOINT}" >> $GITHUB_ENV + echo "S3COMPATSIGV4_SERVICE_NAME=${RUSTFS_SERVICE_NAME}" >> $GITHUB_ENV fi if [ "${S3COMPATSIGV4_INST_ENABLED}" = "true" ]; then INST_ACCESS_KEY="ci$(key_part)" INST_SECRET_KEY="$(secret_part)" INST_BUCKET="${BUCKET_PREFIX}-inst" - # Second bucket is required by setup_minio_buckets.sh but not used for inst storage + # Second bucket is required by setup_rustfs_buckets.sh but not used for inst storage INST_BUCKET_DUMMY="${BUCKET_PREFIX}-inst2" - .github/scripts/setup_minio_buckets.sh \ + .github/scripts/setup_rustfs_buckets.sh \ ../RDM-osf.io \ - "${MINIO_ALIAS}" \ - "${MINIO_ENDPOINT}" \ - "${MINIO_ROOT_USER}" \ - "${MINIO_ROOT_PASSWORD}" \ + "${RUSTFS_ALIAS}" \ + "${RUSTFS_ENDPOINT}" \ + "${RUSTFS_ACCESS_KEY}" \ + "${RUSTFS_SECRET_KEY}" \ "${INST_ACCESS_KEY}" "${INST_SECRET_KEY}" "${INST_BUCKET}" \ "${INST_ACCESS_KEY}" "${INST_SECRET_KEY}" "${INST_BUCKET_DUMMY}" echo "S3COMPATSIGV4_INST_ACCESS_KEY=${INST_ACCESS_KEY}" >> $GITHUB_ENV echo "S3COMPATSIGV4_INST_SECRET_KEY=${INST_SECRET_KEY}" >> $GITHUB_ENV echo "S3COMPATSIGV4_INST_BUCKET=${INST_BUCKET}" >> $GITHUB_ENV - echo "S3COMPATSIGV4_INST_ENDPOINT=${MINIO_ENDPOINT}" >> $GITHUB_ENV + echo "S3COMPATSIGV4_INST_ENDPOINT=${RUSTFS_ENDPOINT}" >> $GITHUB_ENV fi - - name: Start MinIO trace - if: env.MINIO_ENABLED == 'true' - working-directory: RDM-osf.io - run: | - set -eu - docker-compose run -d --name minio-trace --entrypoint /bin/sh minio-mc -c " - set -euo pipefail - mc alias set ${MINIO_ALIAS} ${MINIO_ENDPOINT} ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} - mc admin trace --json --verbose --errors --path '${S3COMPAT_BUCKET_NAME_1}/*' --path '${S3COMPAT_BUCKET_NAME_2}/*' ${MINIO_ALIAS} - " - - - name: Test endpoint accessibility working-directory: RDM-osf.io run: | @@ -1374,9 +1319,6 @@ jobs: EOF args=(ci.config.yaml base_ci.config.yaml) - if [ "${MINIO_ENABLED}" = "true" ]; then - args+=(--minio) - fi if [ "${JUPYTERHUB_ENABLED}" = "true" ]; then args+=(--jupyterhub) fi @@ -1502,23 +1444,19 @@ jobs: path: e2e-tests/result/ retention-days: 7 - - name: Collect MinIO trace - if: always() && env.MINIO_ENABLED == 'true' + - name: Collect RustFS logs + if: always() && env.RUSTFS_ENABLED == 'true' working-directory: RDM-osf.io run: | - set -eu - if docker ps -a --format '{{.Names}}' | grep -q '^minio-trace$'; then - docker logs minio-trace > ../e2e-tests/minio-trace.log || true - docker rm -f minio-trace || true - fi + docker-compose logs --timestamps rustfs > ../e2e-tests/rustfs-logs.txt 2>&1 - - name: Upload MinIO trace - if: always() && env.MINIO_ENABLED == 'true' + - name: Upload RustFS logs + if: always() && env.RUSTFS_ENABLED == 'true' uses: actions/upload-artifact@v4 with: - name: minio-trace-${{ matrix.test-group.name }} - path: e2e-tests/minio-trace.log - if-no-files-found: ignore + name: rustfs-logs-${{ matrix.test-group.name }} + path: e2e-tests/rustfs-logs.txt + retention-days: 7 - name: Collect WEKO logs if: always() && (env.WEKO_ENABLED == 'true' || env.MIBYO_DB_ENABLED == 'true')