diff --git a/.blazar.yaml b/.blazar.yaml new file mode 100644 index 000000000000..4335a8f33d93 --- /dev/null +++ b/.blazar.yaml @@ -0,0 +1,25 @@ +buildpack: + name: Blazar-Buildpack-Java-oss-fork + +env: + MAVEN_PHASE: "package assembly:single deploy" + HADOOP_DEP_VERSION: "3.3.6-hubspot-SNAPSHOT" + MAVEN_BUILD_ARGS: "-Phadoop-3.0 -Dhadoop.profile=3.0 -Dhadoop-three.version=$HADOOP_DEP_VERSION -Dgpg.skip=true -DskipTests -DdeployAtEnd -pl hbase-assembly -am -T1C" + + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +provides: + - hbase diff --git a/.build-jdk17 b/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/workflows/yetus-general-check.yml b/.github/workflows/yetus-general-check.yml new file mode 100644 index 000000000000..40e9589d4020 --- /dev/null +++ b/.github/workflows/yetus-general-check.yml @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus General Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: {} + +jobs: + general-check: + runs-on: ubuntu-latest + timeout-minutes: 600 + permissions: + contents: read + statuses: write + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + fetch-depth: 0 + persist-credentials: false + + - name: Set up JDK 11 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: '11' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus General Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + CHANGE_TARGET: "${{ github.base_ref }}" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + HADOOP_PROFILE: "3.0" + JAVA8_HOME: "/usr/lib/jvm/java-8" + PATCHDIR: "${{ github.workspace }}/yetus-general-check/output" + PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit" + SET_JAVA_HOME: "/usr/lib/jvm/java-11" + SOURCEDIR: "${{ github.workspace }}/src" + TESTS_FILTER: "checkstyle,javac,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes" + BUILD_THREAD: "4" + SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C" + SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C" + BRANCH_NAME: "${{ github.base_ref }}" + DEBUG: 'true' + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-general-check/output >> $GITHUB_STEP_SUMMARY + + - name: Publish Test Results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: yetus-general-check-output + path: ${{ github.workspace }}/yetus-general-check/output + retention-days: 7 + + zizmor: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Check for workflow changes + id: changes + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh pr diff "${{ github.event.pull_request.number }}" --repo "${{ github.repository }}" --name-only | grep -q '^\.github/workflows/'; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout HBase + if: steps.changes.outputs.changed == 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Run zizmor + if: steps.changes.outputs.changed == 'true' + run: pipx run zizmor --min-severity=medium .github/workflows/ diff --git a/.github/workflows/yetus-jdk11-hadoop3-compile-check.yml b/.github/workflows/yetus-jdk11-hadoop3-compile-check.yml new file mode 100644 index 000000000000..603066ff5e5d --- /dev/null +++ b/.github/workflows/yetus-jdk11-hadoop3-compile-check.yml @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus JDK11 Hadoop3 Compile Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: {} + +jobs: + jdk11-hadoop3-compile-check: + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + statuses: write + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + fetch-depth: 0 + persist-credentials: false + + - name: Set up JDK 11 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: '11' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus JDK11 Hadoop3 Compile Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + HADOOP_PROFILE: "3.0" + JAVA8_HOME: "/usr/lib/jvm/java-8" + PATCHDIR: "${{ github.workspace }}/yetus-jdk11-hadoop3-compile-check/output" + PLUGINS: "compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars" + SET_JAVA_HOME: "/usr/lib/jvm/java-11" + SOURCEDIR: "${{ github.workspace }}/src" + TESTS_FILTER: "javac,javadoc" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + BUILD_THREAD: "4" + BRANCH_NAME: "${{ github.base_ref }}" + CHANGE_TARGET: "${{ github.base_ref }}" + SKIP_ERRORPRONE: 'true' + DEBUG: 'true' + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-jdk11-hadoop3-compile-check/output >> $GITHUB_STEP_SUMMARY + + - name: Publish Results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: yetus-jdk11-hadoop3-compile-check-output + path: ${{ github.workspace }}/yetus-jdk11-hadoop3-compile-check/output + retention-days: 7 diff --git a/.github/workflows/yetus-jdk11-hadoop3-unit-check.yml b/.github/workflows/yetus-jdk11-hadoop3-unit-check.yml new file mode 100644 index 000000000000..627f610c95f5 --- /dev/null +++ b/.github/workflows/yetus-jdk11-hadoop3-unit-check.yml @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus JDK11 Hadoop3 Unit Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: {} + +jobs: + jdk11-hadoop3-unit-check: + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + contents: read + statuses: write + + strategy: + fail-fast: false + matrix: + include: + - name: "small" + test_profile: "runSmallTests" + - name: "medium" + test_profile: "runMediumTests" + # Large tests split alphabetically by class name (after "Test" prefix) + # Wave 1: Test[A-H]*, Wave 2: Test[I-R]*, Wave 3: Test[S-Z]* + - name: "large-wave-1" + test_profile: "runLargeTests-wave1" + - name: "large-wave-2" + test_profile: "runLargeTests-wave2" + - name: "large-wave-3" + test_profile: "runLargeTests-wave3" + + name: ${{ matrix.name }} + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + fetch-depth: 0 + persist-credentials: false + + - name: Set up JDK 11 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: '11' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus JDK11 Hadoop3 Unit Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + HADOOP_PROFILE: "3.0" + JAVA8_HOME: "/usr/lib/jvm/java-8" + PATCHDIR: "${{ github.workspace }}/yetus-jdk11-hadoop3-unit-check/output" + PLUGINS: "github,htmlout,maven,unit" + SET_JAVA_HOME: "/usr/lib/jvm/java-11" + SOURCEDIR: "${{ github.workspace }}/src" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes" + BUILD_THREAD: "4" + SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C" + SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C" + BRANCH_NAME: "${{ github.base_ref }}" + CHANGE_TARGET: "${{ github.base_ref }}" + SKIP_ERRORPRONE: 'true' + DEBUG: 'true' + TEST_PROFILE: ${{ matrix.test_profile }} + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-jdk11-hadoop3-unit-check/output >> $GITHUB_STEP_SUMMARY + + - name: Publish Test Results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: yetus-jdk11-hadoop3-unit-check-${{ matrix.name }} + path: ${{ github.workspace }}/yetus-jdk11-hadoop3-unit-check/output + retention-days: 7 diff --git a/.github/workflows/yetus-jdk17-hadoop3-compile-check.yml b/.github/workflows/yetus-jdk17-hadoop3-compile-check.yml new file mode 100644 index 000000000000..17abf8990d9c --- /dev/null +++ b/.github/workflows/yetus-jdk17-hadoop3-compile-check.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus JDK17 Hadoop3 Compile Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + statuses: write + +jobs: + jdk17-hadoop3-compile-check: + runs-on: ubuntu-latest + timeout-minutes: 60 + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + fetch-depth: 0 + persist-credentials: false + + - name: Set up JDK 17 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: '17' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus JDK17 Hadoop3 Compile Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + HADOOP_PROFILE: "3.0" + JAVA8_HOME: "/usr/lib/jvm/java-8" + PATCHDIR: "${{ github.workspace }}/yetus-jdk17-hadoop3-compile-check/output" + PLUGINS: "compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars" + SET_JAVA_HOME: "/usr/lib/jvm/java-17" + SOURCEDIR: "${{ github.workspace }}/src" + TESTS_FILTER: "javac,javadoc" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + BUILD_THREAD: "4" + BRANCH_NAME: "${{ github.base_ref }}" + CHANGE_TARGET: "${{ github.base_ref }}" + SKIP_ERRORPRONE: 'true' + DEBUG: 'true' + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-jdk17-hadoop3-compile-check/output >> $GITHUB_STEP_SUMMARY + + - name: Publish Results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: yetus-jdk17-hadoop3-compile-check-output + path: ${{ github.workspace }}/yetus-jdk17-hadoop3-compile-check/output + retention-days: 7 diff --git a/.github/workflows/yetus-jdk17-hadoop3-unit-check.yml b/.github/workflows/yetus-jdk17-hadoop3-unit-check.yml new file mode 100644 index 000000000000..571b6155dc37 --- /dev/null +++ b/.github/workflows/yetus-jdk17-hadoop3-unit-check.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus JDK17 Hadoop3 Unit Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + statuses: write + +jobs: + jdk17-hadoop3-unit-check: + runs-on: ubuntu-latest + timeout-minutes: 360 + + strategy: + fail-fast: false + matrix: + include: + - name: "small" + test_profile: "runSmallTests" + - name: "medium" + test_profile: "runMediumTests" + # Large tests split alphabetically by class name (after "Test" prefix) + # Wave 1: Test[A-H]*, Wave 2: Test[I-R]*, Wave 3: Test[S-Z]* + - name: "large-wave-1" + test_profile: "runLargeTests-wave1" + - name: "large-wave-2" + test_profile: "runLargeTests-wave2" + - name: "large-wave-3" + test_profile: "runLargeTests-wave3" + + name: ${{ matrix.name }} + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + fetch-depth: 0 + persist-credentials: false + + - name: Set up JDK 17 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: '17' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus JDK17 Hadoop3 Unit Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + HADOOP_PROFILE: "3.0" + JAVA8_HOME: "/usr/lib/jvm/java-8" + PATCHDIR: "${{ github.workspace }}/yetus-jdk17-hadoop3-unit-check/output" + PLUGINS: "github,htmlout,maven,unit" + SET_JAVA_HOME: "/usr/lib/jvm/java-17" + SOURCEDIR: "${{ github.workspace }}/src" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes" + BUILD_THREAD: "4" + SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C" + SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C" + BRANCH_NAME: "${{ github.base_ref }}" + CHANGE_TARGET: "${{ github.base_ref }}" + SKIP_ERRORPRONE: 'true' + DEBUG: 'true' + TEST_PROFILE: ${{ matrix.test_profile }} + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-jdk17-hadoop3-unit-check/output >> $GITHUB_STEP_SUMMARY + + - name: Publish Test Results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: yetus-jdk17-hadoop3-unit-check-${{ matrix.name }} + path: ${{ github.workspace }}/yetus-jdk17-hadoop3-unit-check/output + retention-days: 7 diff --git a/.github/workflows/yetus-jdk8-hadoop2-compile-check.yml b/.github/workflows/yetus-jdk8-hadoop2-compile-check.yml new file mode 100644 index 000000000000..26b27d850356 --- /dev/null +++ b/.github/workflows/yetus-jdk8-hadoop2-compile-check.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus JDK8 Hadoop2 Compile Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: {} + +jobs: + jdk8-hadoop2-compile-check: + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + statuses: write + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + fetch-depth: 0 + persist-credentials: false + + - name: Set up JDK 8 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: '8' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus JDK8 Hadoop2 Compile Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + JAVA8_HOME: "/usr/lib/jvm/java-8" + PATCHDIR: "${{ github.workspace }}/yetus-jdk8-hadoop2-compile-check/output" + PLUGINS: "compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars" + SET_JAVA_HOME: "/usr/lib/jvm/java-8" + SOURCEDIR: "${{ github.workspace }}/src" + TESTS_FILTER: "javac,javadoc" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + BUILD_THREAD: "4" + BRANCH_NAME: "${{ github.base_ref }}" + CHANGE_TARGET: "${{ github.base_ref }}" + SKIP_ERRORPRONE: 'true' + DEBUG: 'true' + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-jdk8-hadoop2-compile-check/output >> $GITHUB_STEP_SUMMARY + + - name: Publish Results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: yetus-jdk8-hadoop2-compile-check-output + path: ${{ github.workspace }}/yetus-jdk8-hadoop2-compile-check/output + retention-days: 7 diff --git a/.github/workflows/yetus-jdk8-hadoop2-unit-check.yml b/.github/workflows/yetus-jdk8-hadoop2-unit-check.yml new file mode 100644 index 000000000000..60f040480db9 --- /dev/null +++ b/.github/workflows/yetus-jdk8-hadoop2-unit-check.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus JDK8 Hadoop2 Unit Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: {} + +jobs: + jdk8-hadoop2-unit-check: + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + contents: read + statuses: write + + strategy: + fail-fast: false + matrix: + include: + - name: "small" + test_profile: "runSmallTests" + - name: "medium" + test_profile: "runMediumTests" + # Large tests split alphabetically by class name (after "Test" prefix) + # Wave 1: Test[A-H]*, Wave 2: Test[I-R]*, Wave 3: Test[S-Z]* + - name: "large-wave-1" + test_profile: "runLargeTests-wave1" + - name: "large-wave-2" + test_profile: "runLargeTests-wave2" + - name: "large-wave-3" + test_profile: "runLargeTests-wave3" + + name: ${{ matrix.name }} + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + fetch-depth: 0 + persist-credentials: false + + - name: Set up JDK 8 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: '8' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus JDK8 Hadoop2 Unit Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + JAVA8_HOME: "/usr/lib/jvm/java-8" + PATCHDIR: "${{ github.workspace }}/yetus-jdk8-hadoop2-unit-check/output" + PLUGINS: "github,htmlout,maven,unit" + SET_JAVA_HOME: "/usr/lib/jvm/java-8" + SOURCEDIR: "${{ github.workspace }}/src" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes" + BUILD_THREAD: "4" + SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C" + SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C" + BRANCH_NAME: "${{ github.base_ref }}" + CHANGE_TARGET: "${{ github.base_ref }}" + SKIP_ERRORPRONE: 'true' + DEBUG: 'true' + TEST_PROFILE: ${{ matrix.test_profile }} + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Publish Job Summary + if: always() + run: | + cd "${{ github.workspace }}" + python3 src/dev-support/yetus_console_to_md.py yetus-jdk8-hadoop2-unit-check/output >> $GITHUB_STEP_SUMMARY + + - name: Publish Test Results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: yetus-jdk8-hadoop2-unit-check-${{ matrix.name }} + path: ${{ github.workspace }}/yetus-jdk8-hadoop2-unit-check/output + retention-days: 7 diff --git a/.gitignore b/.gitignore index 666771f7e251..9ab7a972d7ac 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,8 @@ .DS_Store .classpath /build -/.idea/ +.idea/* +!.idea/vcs.xml /logs *target/ *.orig @@ -24,3 +25,4 @@ linklint/ tmp **/.flattened-pom.xml .vscode/ +**/__pycache__ diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000000..b2aa0751c813 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,39 @@ + + + + + + + + + + diff --git a/CHANGES.md b/CHANGES.md index debb62e9a2ad..458f125200be 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,314 @@ --> # HBASE Changelog +## Release 2.6.5 - 2026-03-20 + + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-30003](https://issues.apache.org/jira/browse/HBASE-30003) | Upgrade deprecated GitHub Actions versions for branch-2.x | Major | build | +| [HBASE-30002](https://issues.apache.org/jira/browse/HBASE-30002) | Rewrite TestExportSnapshot related UTs | Major | mapreduce, test | +| [HBASE-29999](https://issues.apache.org/jira/browse/HBASE-29999) | Update the download links in the README | Minor | . | +| [HBASE-29994](https://issues.apache.org/jira/browse/HBASE-29994) | Separate backwards compatibility checks to a new jenkins job | Major | jenkins, scripts | +| [HBASE-29981](https://issues.apache.org/jira/browse/HBASE-29981) | Upgrade hbase-endpoint to use junit5 | Major | test | +| [HBASE-29927](https://issues.apache.org/jira/browse/HBASE-29927) | Upgrade hbase-external-blockcache to use junit5 | Major | test | +| [HBASE-29988](https://issues.apache.org/jira/browse/HBASE-29988) | Upgrade hbase-archetypes to use junit5 | Major | test | +| [HBASE-29969](https://issues.apache.org/jira/browse/HBASE-29969) | Bump lz4-java 1.10.4 | Major | dependencies | +| [HBASE-29982](https://issues.apache.org/jira/browse/HBASE-29982) | Upgrade TestTags to JUnit5 and polish the code | Major | test | +| [HBASE-29949](https://issues.apache.org/jira/browse/HBASE-29949) | Downgrade info log in GlobalMetricRegistriesAdapter | Major | metrics | +| [HBASE-29978](https://issues.apache.org/jira/browse/HBASE-29978) | Remove redundant imports in regionserver.jsp file | Trivial | UI | +| [HBASE-29954](https://issues.apache.org/jira/browse/HBASE-29954) | Support magic links to PR and JIRA in IDEA | Minor | community | +| [HBASE-29980](https://issues.apache.org/jira/browse/HBASE-29980) | Polish jenkinsfiles after splitting integration test out | Major | jenkins | +| [HBASE-28601](https://issues.apache.org/jira/browse/HBASE-28601) | Enable setting memstore on-heap sizes in bytes | Major | regionserver | +| [HBASE-29966](https://issues.apache.org/jira/browse/HBASE-29966) | Fix race in FlushRegionProcedure and rewrite TestAcidGuarantees | Major | proc-v2, test | +| [HBASE-29979](https://issues.apache.org/jira/browse/HBASE-29979) | Remove JUnit5 related dependencies in hbase-protocol-shaded | Major | test | +| [HBASE-29967](https://issues.apache.org/jira/browse/HBASE-29967) | Upgrade hbase-backup to use junit5 | Major | backup&restore, test | +| [HBASE-29938](https://issues.apache.org/jira/browse/HBASE-29938) | Upgrade hbase-it to use junit5 | Major | integration tests, test | +| [HBASE-29940](https://issues.apache.org/jira/browse/HBASE-29940) | GC Collector stats can't be displayed properly in JDK21 | Major | UI | +| [HBASE-29939](https://issues.apache.org/jira/browse/HBASE-29939) | Upgrade hbase-logging to use junit5 | Major | logging, test | +| [HBASE-29937](https://issues.apache.org/jira/browse/HBASE-29937) | Upgrade hbase-metrics-api to use junit5 | Major | metrics, test | +| [HBASE-29934](https://issues.apache.org/jira/browse/HBASE-29934) | Quota Info in Table Attributes should display more clearly | Major | Quotas, UI | +| [HBASE-29935](https://issues.apache.org/jira/browse/HBASE-29935) | Update copyright notices year to 2026 | Minor | documentation | +| [HBASE-29926](https://issues.apache.org/jira/browse/HBASE-29926) | Upgrade hbase-hadoop-compat to use junit5 | Major | test | +| [HBASE-29932](https://issues.apache.org/jira/browse/HBASE-29932) | Upgrade hbase-http to use junit5 | Major | test | +| [HBASE-23644](https://issues.apache.org/jira/browse/HBASE-23644) | Thrift2 DemoClient.py is not running | Minor | . | +| [HBASE-29457](https://issues.apache.org/jira/browse/HBASE-29457) | thrift1/DemoClient.py is not running | Minor | . | +| [HBASE-29910](https://issues.apache.org/jira/browse/HBASE-29910) | Upgrade hbase-client to use junit5 | Major | Client, test | +| [HBASE-29925](https://issues.apache.org/jira/browse/HBASE-29925) | TestCheckTestClasses fails on branch-2.x with JDK8 | Major | test | +| [HBASE-29874](https://issues.apache.org/jira/browse/HBASE-29874) | Removing the useless parameters from ScanQueryMatcher.isCellTTLExpired | Major | . | +| [HBASE-29919](https://issues.apache.org/jira/browse/HBASE-29919) | Test case TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth failed | Major | test | +| [HBASE-29911](https://issues.apache.org/jira/browse/HBASE-29911) | Upgrade hbase-examples to use junit5 | Major | test | +| [HBASE-29871](https://issues.apache.org/jira/browse/HBASE-29871) | During WAL write times out add DataNode address in. the exception message | Minor | Filesystem Integration, io | +| [HBASE-29902](https://issues.apache.org/jira/browse/HBASE-29902) | Upgrade hbase-thrift to use junit5 | Major | test, Thrift | +| [HBASE-29887](https://issues.apache.org/jira/browse/HBASE-29887) | Upgrade hbase-hbtop to use junit5 | Major | hbtop, test | +| [HBASE-29721](https://issues.apache.org/jira/browse/HBASE-29721) | Add validation for Mutations without column families | Minor | Client | +| [HBASE-29901](https://issues.apache.org/jira/browse/HBASE-29901) | Upgrade hbase-zookeeper to use junit5 | Major | test, Zookeeper | +| [HBASE-29898](https://issues.apache.org/jira/browse/HBASE-29898) | Upgrade hbase-replication to use junit5 | Major | Replication, test | +| [HBASE-29903](https://issues.apache.org/jira/browse/HBASE-29903) | hbtop should display default replica id | Major | hbtop | +| [HBASE-29885](https://issues.apache.org/jira/browse/HBASE-29885) | Enable Disabled UT TestNamespacesInstanceResource.testInvalidNamespacePostsAndPuts | Minor | REST, test | +| [HBASE-29900](https://issues.apache.org/jira/browse/HBASE-29900) | Correct logger initialization under hbase-procedure | Major | proc-v2, test | +| [HBASE-29844](https://issues.apache.org/jira/browse/HBASE-29844) | Upgrade hbase-metrics to use junit5 | Major | metrics, test | +| [HBASE-29562](https://issues.apache.org/jira/browse/HBASE-29562) | Add RegionServer Metrics for excluded DataNodes | Minor | metrics | +| [HBASE-29879](https://issues.apache.org/jira/browse/HBASE-29879) | Upgrade hbase-procedure to use junit5 | Major | . | +| [HBASE-29876](https://issues.apache.org/jira/browse/HBASE-29876) | Upgrade hbase-shell to use junit5 | Major | shell, test | +| [HBASE-25508](https://issues.apache.org/jira/browse/HBASE-25508) | Add a python example of using the thrift proxy in thrift-over-http mode | Minor | Thrift | +| [HBASE-29858](https://issues.apache.org/jira/browse/HBASE-29858) | Upgrade hbase-rest to use junit5 | Major | test | +| [HBASE-29867](https://issues.apache.org/jira/browse/HBASE-29867) | Promote some medium tests to large tests | Major | test | +| [HBASE-29866](https://issues.apache.org/jira/browse/HBASE-29866) | Generate job summary for our yetus check github action | Major | build | +| [HBASE-29845](https://issues.apache.org/jira/browse/HBASE-29845) | Description of the hbase:slowlog is not displayed correctly | Major | UI | +| [HBASE-29820](https://issues.apache.org/jira/browse/HBASE-29820) | Bump the syntax of velocity template to align with velocity 2.x syntax | Minor | . | +| [HBASE-29828](https://issues.apache.org/jira/browse/HBASE-29828) | Upgrade TestIPC related tests to junit5 | Major | rpc, test | +| [HBASE-29821](https://issues.apache.org/jira/browse/HBASE-29821) | Fix maven-source-plugin duplicate jar failure | Minor | . | +| [HBASE-29819](https://issues.apache.org/jira/browse/HBASE-29819) | Upgrade hbase-asyncfs to use junit5 | Major | Filesystem Integration, test | +| [HBASE-29817](https://issues.apache.org/jira/browse/HBASE-29817) | Stop retrying on DoNotRetryIOException when waiting procedure result | Major | Admin, asyncclient | +| [HBASE-29727](https://issues.apache.org/jira/browse/HBASE-29727) | Introduce a String pool for repeating filename, region and cf string fields in BlockCacheKey | Major | . | +| [HBASE-29808](https://issues.apache.org/jira/browse/HBASE-29808) | Simplify backup history retrieval | Minor | backup&restore | +| [HBASE-29810](https://issues.apache.org/jira/browse/HBASE-29810) | Polish TestFileChangeWatcher | Major | test | +| [HBASE-29796](https://issues.apache.org/jira/browse/HBASE-29796) | Allow sleepForRetry replication config to be overridden by replication peers | Minor | Replication | +| [HBASE-29794](https://issues.apache.org/jira/browse/HBASE-29794) | Only exclude tests when its failure rate above a threshold | Major | flakies, jenkins | +| [HBASE-29706](https://issues.apache.org/jira/browse/HBASE-29706) | Modify table with lazy mode should pass if coprocessors have not changed | Major | . | +| [HBASE-29732](https://issues.apache.org/jira/browse/HBASE-29732) | Add hbase.rs.evictblocksonsplit to UI and CacheConfig logs | Major | BlockCache | +| [HBASE-29729](https://issues.apache.org/jira/browse/HBASE-29729) | Add per-region table descriptor hash to RegionServer JMX metrics | Minor | metrics, regionserver | +| [HBASE-29767](https://issues.apache.org/jira/browse/HBASE-29767) | Fix IOTimePerHit NaN issue in BucketCacheStats | Minor | BlockCache | +| [HBASE-29645](https://issues.apache.org/jira/browse/HBASE-29645) | AsyncBufferedMutatorImpl concurrency improvement | Major | Client | +| [HBASE-29731](https://issues.apache.org/jira/browse/HBASE-29731) | Add L1 and L2 cache hit/miss caching metrics | Trivial | BlockCache | +| [HBASE-29725](https://issues.apache.org/jira/browse/HBASE-29725) | Refactor the testScan method of TestDefaultMemStore | Minor | in-memory-compaction | +| [HBASE-29714](https://issues.apache.org/jira/browse/HBASE-29714) | Increase DEFAULT\_RS\_REMOTE\_PROC\_RETRY\_LIMIT to 10 | Major | proc-v2 | +| [HBASE-29033](https://issues.apache.org/jira/browse/HBASE-29033) | Add a shell command for inspecting the state of enable/disable\_rpc\_throttle | Major | Quotas, shell | +| [HBASE-29703](https://issues.apache.org/jira/browse/HBASE-29703) | Remove duplicate calls to withNextBlockOnDiskSize | Minor | BucketCache | +| [HBASE-29473](https://issues.apache.org/jira/browse/HBASE-29473) | Obtain target cluster's token for cross clusters job | Major | . | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29955](https://issues.apache.org/jira/browse/HBASE-29955) | HMaster getting aborted due to NPE while creating snapshot for invalid table name | Major | snapshots | +| [HBASE-29987](https://issues.apache.org/jira/browse/HBASE-29987) | Replication position corruption when WAL file switch detected in ReplicationSourceWALReader run loop | Minor | Replication, wal, Zookeeper | +| [HBASE-29800](https://issues.apache.org/jira/browse/HBASE-29800) | WAL logs are unprotected during first full backup | Major | backup&restore | +| [HBASE-29977](https://issues.apache.org/jira/browse/HBASE-29977) | TestFSUtils fails with class not found | Major | . | +| [HBASE-29777](https://issues.apache.org/jira/browse/HBASE-29777) | Slow and Large Response RPC StartTime in SlowLog Operation Details page is incorrect | Major | . | +| [HBASE-29834](https://issues.apache.org/jira/browse/HBASE-29834) | MILLIS\_BETWEEN\_NEXTS metric is not updated on branch-3+ | Minor | asyncclient, Client, scan | +| [HBASE-29759](https://issues.apache.org/jira/browse/HBASE-29759) | TestAsyncTableScan fails with tracing assert error | Major | asyncclient, tracing | +| [HBASE-29862](https://issues.apache.org/jira/browse/HBASE-29862) | Test case TestClearRegionBlockCache#testClearBlockCache failed | Major | BlockCache, test | +| [HBASE-28913](https://issues.apache.org/jira/browse/HBASE-28913) | LoadBalancerPerformanceEvaluation fails with NPE | Major | tooling | +| [HBASE-29907](https://issues.apache.org/jira/browse/HBASE-29907) | ROWCOL bloom filter + StoreScanner.trySkipToNextColumn can surface out-of-order cells, causing read failure “isDelete failed” | Blocker | Filters, Scanners | +| [HBASE-29915](https://issues.apache.org/jira/browse/HBASE-29915) | Rewrite TestMultiRowResource and TestSchemaResource | Major | REST, test | +| [HBASE-29917](https://issues.apache.org/jira/browse/HBASE-29917) | TestCheckTestClasses fails with JDK8 + hadoop3 | Major | test | +| [HBASE-28985](https://issues.apache.org/jira/browse/HBASE-28985) | Fix memStoreSize metric for a table and add heap and off-heap size metrics | Minor | metrics | +| [HBASE-29886](https://issues.apache.org/jira/browse/HBASE-29886) | list\_namespace will throw instance vars on non-persistent Java type warning | Minor | shell | +| [HBASE-29846](https://issues.apache.org/jira/browse/HBASE-29846) | Fix backup history ordering | Minor | backup&restore | +| [HBASE-29877](https://issues.apache.org/jira/browse/HBASE-29877) | Load Test Tool doesn't read with TIMELINE consistency, because it sets replica id explicitly | Major | test, util | +| [HBASE-29859](https://issues.apache.org/jira/browse/HBASE-29859) | Initialization log for DataTieringManager is incorrect | Minor | regionserver | +| [HBASE-29896](https://issues.apache.org/jira/browse/HBASE-29896) | Raw scan incorrectly skips cells expired by cell-level TTL | Minor | Scanners | +| [HBASE-29870](https://issues.apache.org/jira/browse/HBASE-29870) | Correct the UI Column name from Stats to State in rsgroup.jsp | Major | UI | +| [HBASE-29868](https://issues.apache.org/jira/browse/HBASE-29868) | Can not start mini dfs cluster when building against hadoop 3.3.x | Major | hadoop3, logging | +| [HBASE-29857](https://issues.apache.org/jira/browse/HBASE-29857) | BucketCache fails to start when persistence file was written with empty cache | Critical | regionserver | +| [HBASE-29831](https://issues.apache.org/jira/browse/HBASE-29831) | Region replica's can fall behind due to a NPE during processing | Minor | read replicas | +| [HBASE-29860](https://issues.apache.org/jira/browse/HBASE-29860) | Fix NullPointerException on Maven site build | Blocker | build, site | +| [HBASE-29854](https://issues.apache.org/jira/browse/HBASE-29854) | Add osgi annotations dependency when generating javadoc | Major | build, site | +| [HBASE-29788](https://issues.apache.org/jira/browse/HBASE-29788) | Bump the version for Maven Project Info Reports Plugin | Major | build, pom, site | +| [HBASE-29141](https://issues.apache.org/jira/browse/HBASE-29141) | Calculate default maxQueueLength call queues correctly | Major | master, regionserver | +| [HBASE-29830](https://issues.apache.org/jira/browse/HBASE-29830) | Manage org.bouncycastle:bcutil-jdk18on version | Major | . | +| [HBASE-29836](https://issues.apache.org/jira/browse/HBASE-29836) | Add org.bouncycastle:bcpkix-jdk18on as explicit test dependency to hbase-http | Major | test | +| [HBASE-29783](https://issues.apache.org/jira/browse/HBASE-29783) | Fix flaky TestVerifyBucketCacheFile.testRetrieveFromFile test | Minor | . | +| [HBASE-29144](https://issues.apache.org/jira/browse/HBASE-29144) | Client request fails for KERBEROS with RpcConnectionRegistry | Blocker | Client, rpc, security | +| [HBASE-29802](https://issues.apache.org/jira/browse/HBASE-29802) | NPE when shutting down mini cluster cause tests hang | Major | mapreduce, test | +| [HBASE-29795](https://issues.apache.org/jira/browse/HBASE-29795) | RegionServer abort because of NPE when closing compacted store file | Major | Compaction, regionserver | +| [HBASE-29793](https://issues.apache.org/jira/browse/HBASE-29793) | TestDefaultHandlerUsageQuota fails 100% on flaky dashboard | Major | Quotas, test | +| [HBASE-29797](https://issues.apache.org/jira/browse/HBASE-29797) | Should not create wal directory when creating WAL instance | Blocker | Region Assignment | +| [HBASE-29775](https://issues.apache.org/jira/browse/HBASE-29775) | Allow inspecting log levels in Master UI in read-only mode | Minor | UI | +| [HBASE-29742](https://issues.apache.org/jira/browse/HBASE-29742) | Compaction scan returns single cells instead of rows after 10MB | Major | Compaction | +| [HBASE-29744](https://issues.apache.org/jira/browse/HBASE-29744) | Data loss scenario for WAL files belonging to RS added between backups | Major | backup&restore | +| [HBASE-29526](https://issues.apache.org/jira/browse/HBASE-29526) | Dynamic configuration not working for coprocessor | Major | Coprocessors | +| [HBASE-29771](https://issues.apache.org/jira/browse/HBASE-29771) | TestThreadHandlerUsageQuota fails on flaky dashboard | Major | Quotas | +| [HBASE-29586](https://issues.apache.org/jira/browse/HBASE-29586) | Setting \`hbase.oldwals.cleaner.thread.size\` to negative value will throw IllegalArgumentException | Minor | . | +| [HBASE-29760](https://issues.apache.org/jira/browse/HBASE-29760) | TestIncrementalBackup fails 100% on flaky dashboard | Major | backup&restore | +| [HBASE-29501](https://issues.apache.org/jira/browse/HBASE-29501) | IOException in SerialReplicationChecker.canPush causes entries to be pushed out of order | Major | Replication | +| [HBASE-28158](https://issues.apache.org/jira/browse/HBASE-28158) | Decouple RIT list management from TRSP invocation | Major | master, Region Assignment | +| [HBASE-29709](https://issues.apache.org/jira/browse/HBASE-29709) | NPE may occur when compacting table via RawAsyncHBaseAdmin | Minor | asyncclient | +| [HBASE-29698](https://issues.apache.org/jira/browse/HBASE-29698) | Correct log message in ReplicationSourceShipper.clearWALEntryBatch | Major | Replication | +| [HBASE-29145](https://issues.apache.org/jira/browse/HBASE-29145) | Table Stats shows store file size as zero always for hbase:meta table | Major | UI | +| [HBASE-29707](https://issues.apache.org/jira/browse/HBASE-29707) | Fix region cache % metrics miss calculation | Major | BucketCache | +| [HBASE-29701](https://issues.apache.org/jira/browse/HBASE-29701) | Update README.txt in hbase-protocol-shaded | Minor | . | +| [HBASE-29704](https://issues.apache.org/jira/browse/HBASE-29704) | Replace unsupported forkMode failsafe parameter in hbase-it | Major | integration tests | +| [HBASE-29702](https://issues.apache.org/jira/browse/HBASE-29702) | Remove shade plugin from hbase-protocol-shaded | Minor | . | +| [HBASE-29667](https://issues.apache.org/jira/browse/HBASE-29667) | The block priority is initialized as MULTI when the data block is first written into the BucketCache | Major | BucketCache | +| [HBASE-29700](https://issues.apache.org/jira/browse/HBASE-29700) | Always close RPC servers in AbstractTestIPC | Minor | test | +| [HBASE-29686](https://issues.apache.org/jira/browse/HBASE-29686) | Compatible issue of HFileOutputFormat2#configureRemoteCluster | Minor | . | +| [HBASE-28839](https://issues.apache.org/jira/browse/HBASE-28839) | Exception handling during retrieval of bucket-cache from persistence. | Major | BucketCache | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29923](https://issues.apache.org/jira/browse/HBASE-29923) | Polish the nightly jenkins file | Major | jenkins, scripts | +| [HBASE-29922](https://issues.apache.org/jira/browse/HBASE-29922) | Change hadoop check versions after supporting hadoop 3.4.3 | Major | scripts | +| [HBASE-29921](https://issues.apache.org/jira/browse/HBASE-29921) | Bump default hadoop 3 version to 3.4.3 | Major | dependencies, hadoop3 | +| [HBASE-29807](https://issues.apache.org/jira/browse/HBASE-29807) | Depend on jcip-annotations explicitly | Major | build, dependencies, hadoop3 | +| [HBASE-29569](https://issues.apache.org/jira/browse/HBASE-29569) | Implement a built-in TieringValueProvider for parsing the date value from the rowkey | Minor | . | +| [HBASE-29837](https://issues.apache.org/jira/browse/HBASE-29837) | Backport HBASE-27355 Separate meta read requests from master and client to branch-2 | Minor | . | +| [HBASE-29596](https://issues.apache.org/jira/browse/HBASE-29596) | Migrate Canary Status Jamon page back to JSP | Major | UI | +| [HBASE-29531](https://issues.apache.org/jira/browse/HBASE-29531) | Migrate RegionServer Status Jamon page back to JSP | Major | UI | +| [HBASE-29223](https://issues.apache.org/jira/browse/HBASE-29223) | Migrate Master Status Jamon page back to JSP | Major | UI | +| [HBASE-29568](https://issues.apache.org/jira/browse/HBASE-29568) | Allow for a configurable "grace period" when using Time Based Priority | Major | . | + + +### OTHER: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29582](https://issues.apache.org/jira/browse/HBASE-29582) | Migrate Jamon pages back to JSP | Major | UI | +| [HBASE-29985](https://issues.apache.org/jira/browse/HBASE-29985) | Bump org.apache.zookeeper:zookeeper from 3.8.4 to 3.8.6 | Major | dependabot, security, Zookeeper | +| [HBASE-29971](https://issues.apache.org/jira/browse/HBASE-29971) | Upgrade to hbase-thirdparty 4.1.13 | Major | dependencies, thirdparty | +| [HBASE-29893](https://issues.apache.org/jira/browse/HBASE-29893) | Add zizmor for GitHub Actions workflows security analysis | Major | build | +| [HBASE-29930](https://issues.apache.org/jira/browse/HBASE-29930) | Separate packaging and integration check in nightly job to new jenkins job | Major | jenkins | +| [HBASE-29454](https://issues.apache.org/jira/browse/HBASE-29454) | Update hbase-examples scripts to be compatible with Python 3 | Minor | . | +| [HBASE-29928](https://issues.apache.org/jira/browse/HBASE-29928) | Bump io.airlift:aircompressor from 0.27 to 2.0.3 | Major | compression, dependabot, dependencies, security | +| [HBASE-29916](https://issues.apache.org/jira/browse/HBASE-29916) | Correct alter\_async help text examples | Trivial | shell | +| [HBASE-29897](https://issues.apache.org/jira/browse/HBASE-29897) | Bump org.apache.avro:avro from 1.11.4 to 1.11.5 | Major | dependabot, security | +| [HBASE-29873](https://issues.apache.org/jira/browse/HBASE-29873) | Improve the job summary for unit check | Major | build, test | +| [HBASE-29888](https://issues.apache.org/jira/browse/HBASE-29888) | GitHub Actions job summary should show which tests failed | Major | build | +| [HBASE-29838](https://issues.apache.org/jira/browse/HBASE-29838) | Run Hadoop Check as a GitHub Action | Major | build | +| [HBASE-29792](https://issues.apache.org/jira/browse/HBASE-29792) | Bump org.apache.logging.log4j:log4j-core from 2.17.2 to 2.25.3 | Major | dependabot, dependencies, logging, security | +| [HBASE-29855](https://issues.apache.org/jira/browse/HBASE-29855) | Bump maven-site-plugin to 3.21.0 | Blocker | build, site | +| [HBASE-29851](https://issues.apache.org/jira/browse/HBASE-29851) | Add more directories to yetus skip-dirs | Major | build | +| [HBASE-29852](https://issues.apache.org/jira/browse/HBASE-29852) | GHA general check broken on branch-2\* | Major | build | +| [HBASE-29829](https://issues.apache.org/jira/browse/HBASE-29829) | Pagination support for Hide Old Comments in PRs | Major | build | +| [HBASE-29787](https://issues.apache.org/jira/browse/HBASE-29787) | Run Yetus General Checks as a GitHub Action | Major | build | +| [HBASE-29791](https://issues.apache.org/jira/browse/HBASE-29791) | Backport 'HBASE-29761: The HBase UI's Debug Dump is not redacting sensitive information' to branch-2.6 | Critical | security, UI | +| [HBASE-28804](https://issues.apache.org/jira/browse/HBASE-28804) | Implement asynchronous retrieval of bucket-cache data from persistence. | Major | BucketCache | +| [HBASE-29743](https://issues.apache.org/jira/browse/HBASE-29743) | Fix typos in documentation for BulkLoadObserver | Trivial | documentation | +| [HBASE-29740](https://issues.apache.org/jira/browse/HBASE-29740) | Upgrade lz4-java to 1.8.1+ | Major | security | +| [HBASE-29695](https://issues.apache.org/jira/browse/HBASE-29695) | Fix typo in RestoreRequest's "Builder.withOvewrite()" method | Minor | backup&restore | +| [HBASE-29724](https://issues.apache.org/jira/browse/HBASE-29724) | Backport missing changes of "HBASE-25334: TestRSGroupsFallback.testFallback is flaky" into branch-2 and branch-2.6 | Major | . | +| [HBASE-29723](https://issues.apache.org/jira/browse/HBASE-29723) | Backport "HBASE-25282: Remove processingServers in DeadServer as we can get this information by Procedure of master" to branch-2.6 | Major | . | + + +## Release 2.6.4 - 2025-11-10 + + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-28463](https://issues.apache.org/jira/browse/HBASE-28463) | Time Based Priority for BucketCache | Major | BucketCache | +| [HBASE-28919](https://issues.apache.org/jira/browse/HBASE-28919) | Soft drop for destructive table actions | Major | master, snapshots | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29679](https://issues.apache.org/jira/browse/HBASE-29679) | Suppress stack trace in RpcThrottlingException | Minor | Quotas | +| [HBASE-29663](https://issues.apache.org/jira/browse/HBASE-29663) | TimeBasedLimiters should support dynamic configuration refresh | Major | . | +| [HBASE-29653](https://issues.apache.org/jira/browse/HBASE-29653) | Build fails on riscv64 due to os-maven-plugin not recognizing RISC-V architecture | Major | build | +| [HBASE-29650](https://issues.apache.org/jira/browse/HBASE-29650) | Upgrade tomcat-jasper to 9.0.110 | Major | UI | +| [HBASE-29637](https://issues.apache.org/jira/browse/HBASE-29637) | Implement ResourceCheckerJUnitListener for junit 5 | Major | test | +| [HBASE-29649](https://issues.apache.org/jira/browse/HBASE-29649) | Un-deprecate preWALRestore and postWALRestore in RegionCoprocessorHost | Minor | Coprocessors | +| [HBASE-29636](https://issues.apache.org/jira/browse/HBASE-29636) | Implement TimedOutTestsListener for junit 5 | Major | test | +| [HBASE-29626](https://issues.apache.org/jira/browse/HBASE-29626) | Refactor server side scan metrics for Coproc hooks | Minor | . | +| [HBASE-28440](https://issues.apache.org/jira/browse/HBASE-28440) | Add support for using mapreduce sort in HFileOutputFormat2 | Major | backup&restore | +| [HBASE-29627](https://issues.apache.org/jira/browse/HBASE-29627) | Handle any block cache fetching errors when reading a block in HFileReaderImpl | Major | BlockCache | +| [HBASE-29576](https://issues.apache.org/jira/browse/HBASE-29576) | Replicate HBaseClassTestRule functionality for Junit 5 | Major | test | +| [HBASE-29612](https://issues.apache.org/jira/browse/HBASE-29612) | Remove HBaseTestingUtil.forceChangeTaskLogDir | Major | . | +| [HBASE-29608](https://issues.apache.org/jira/browse/HBASE-29608) | Add test to make sure we do not have copy paste errors in the TAG value | Minor | test | +| [HBASE-29610](https://issues.apache.org/jira/browse/HBASE-29610) | Add and use String constants for Junit 5 @Tag annotations | Minor | integration tests, test | +| [HBASE-29573](https://issues.apache.org/jira/browse/HBASE-29573) | Fully load QuotaCache instead of reading individual rows on demand | Minor | . | +| [HBASE-29571](https://issues.apache.org/jira/browse/HBASE-29571) | Fix Javadoc typo: 'repoen' should be 'reopen' | Trivial | . | +| [HBASE-29575](https://issues.apache.org/jira/browse/HBASE-29575) | Do not limit surefire to Junit 4 | Major | test | +| [HBASE-29496](https://issues.apache.org/jira/browse/HBASE-29496) | Fix Javadoc typo: 'DsiableTableProcedure' should be 'DisableTableProcedure' | Trivial | documentation | +| [HBASE-29494](https://issues.apache.org/jira/browse/HBASE-29494) | Capture Scan RPC processing time and queuing time in Scan Metrics | Minor | . | +| [HBASE-29479](https://issues.apache.org/jira/browse/HBASE-29479) | QuotaCache is not correctly populated until runs of QuotaRefresherChore | Minor | . | +| [HBASE-29556](https://issues.apache.org/jira/browse/HBASE-29556) | Display HBCK and CatalogJanitor report errors properly on HBCK Report page | Major | UI | +| [HBASE-29431](https://issues.apache.org/jira/browse/HBASE-29431) | Update the 'ExcludeDNs' information with the cause in RS UI | Major | UI | +| [HBASE-29528](https://issues.apache.org/jira/browse/HBASE-29528) | Support for cellVisibility in Thrift interface | Minor | Thrift | +| [HBASE-29290](https://issues.apache.org/jira/browse/HBASE-29290) | Include port number of Region Server in the Replication Status message | Minor | shell | +| [HBASE-29469](https://issues.apache.org/jira/browse/HBASE-29469) | Add RPC throttling metrics to RegionServer for quota monitoring | Minor | metrics | +| [HBASE-29508](https://issues.apache.org/jira/browse/HBASE-29508) | Define HBase specific TLS config properties for InfoServer | Major | . | +| [HBASE-29477](https://issues.apache.org/jira/browse/HBASE-29477) | Make TableOutputCommitter Configurable for TableOutputFormat | Blocker | . | +| [HBASE-29481](https://issues.apache.org/jira/browse/HBASE-29481) | Make TLS protocols and include cipher list configurable for HTTPS InfoServer | Major | security, UI | +| [HBASE-15625](https://issues.apache.org/jira/browse/HBASE-15625) | Make minimum values configurable and smaller | Minor | . | +| [HBASE-29467](https://issues.apache.org/jira/browse/HBASE-29467) | Redundant conditions in CostFunction.scale() method | Major | Balancer | +| [HBASE-29450](https://issues.apache.org/jira/browse/HBASE-29450) | Bump org.apache.commons:commons-lang3 from 3.17.0 to 3.18.0 | Major | dependabot, dependencies, security | +| [HBASE-29398](https://issues.apache.org/jira/browse/HBASE-29398) | Server side scan metrics for bytes read from FS vs Block cache vs memstore | Major | . | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29662](https://issues.apache.org/jira/browse/HBASE-29662) | Reading data via TableSnapshotInputFormat should fail instead of reading no data if restore directory got deleted | Critical | snapshots | +| [HBASE-29622](https://issues.apache.org/jira/browse/HBASE-29622) | Flaky Test : TestBackupDelete#testBackupDeleteUpdatesIncrementalBackupSet | Major | backup&restore | +| [HBASE-29677](https://issues.apache.org/jira/browse/HBASE-29677) | Thread safety in QuotaRefresherChore | Minor | . | +| [HBASE-29604](https://issues.apache.org/jira/browse/HBASE-29604) | BackupHFileCleaner uses flawed time based check | Critical | backup&restore | +| [HBASE-29629](https://issues.apache.org/jira/browse/HBASE-29629) | Record the quota user name value on metrics for RpcThrottlingExceptions | Minor | Quotas | +| [HBASE-29623](https://issues.apache.org/jira/browse/HBASE-29623) | Blocks for CFs with BlockCache disabled may still get cached on write or compaction | Major | BlockCache | +| [HBASE-29550](https://issues.apache.org/jira/browse/HBASE-29550) | Reflection error in TestRSGroupsKillRS with Java 21 | Major | test | +| [HBASE-29587](https://issues.apache.org/jira/browse/HBASE-29587) | Set Test category for TestSnapshotProcedureEarlyExpiration | Minor | snapshots | +| [HBASE-29601](https://issues.apache.org/jira/browse/HBASE-29601) | Handle Junit 5 tests in TestCheckTestClasses | Major | test | +| [HBASE-29602](https://issues.apache.org/jira/browse/HBASE-29602) | Add -Djava.security.manager=allow to JDK18+ surefire JVM flags | Major | integration tests, test | +| [HBASE-29548](https://issues.apache.org/jira/browse/HBASE-29548) | Update ApacheDS to 2.0.0.AM27 and ldap-api to 2.1.7 | Major | test | +| [HBASE-29577](https://issues.apache.org/jira/browse/HBASE-29577) | Fix NPE from RegionServerRpcQuotaManager when reloading configuration | Minor | . | +| [HBASE-27157](https://issues.apache.org/jira/browse/HBASE-27157) | Potential race condition in WorkerAssigner | Minor | proc-v2 | +| [HBASE-29566](https://issues.apache.org/jira/browse/HBASE-29566) | TestPrefetch.testPrefetchWithDelay seems flakey | Major | . | +| [HBASE-29453](https://issues.apache.org/jira/browse/HBASE-29453) | NPE on CacheAwareLoadBalancer.balanceTable | Trivial | Balancer | +| [HBASE-29540](https://issues.apache.org/jira/browse/HBASE-29540) | Unhandled IllegalArgumentException in HBase Web UI When Accessing table.jsp with Invalid Table Name | Minor | UI | +| [HBASE-29558](https://issues.apache.org/jira/browse/HBASE-29558) | Broken hbase-shell no-cluster tests | Major | shell | +| [HBASE-29570](https://issues.apache.org/jira/browse/HBASE-29570) | Set no watches on the node when recursively deleting the node and its child nodes | Minor | Zookeeper | +| [HBASE-28881](https://issues.apache.org/jira/browse/HBASE-28881) | Setting \`hbase.master.procedure.threads\` to negative value doesn't break HMaster but clients cannot connect | Critical | master | +| [HBASE-29549](https://issues.apache.org/jira/browse/HBASE-29549) | Mockito failures in TestServerCall with Java 21 | Major | test | +| [HBASE-28866](https://issues.apache.org/jira/browse/HBASE-28866) | Setting \`hbase.oldwals.cleaner.thread.size\` to negative value will break HMaster and produce hard-to-diagnose logs | Critical | master | +| [HBASE-29532](https://issues.apache.org/jira/browse/HBASE-29532) | NPE error when there is EOF for specific recover folder | Major | . | +| [HBASE-29502](https://issues.apache.org/jira/browse/HBASE-29502) | RegionReplicaReplicationEndpoint fails to forward mutations when meta cache does not contain secondary replica locations | Major | read replicas | +| [HBASE-29544](https://issues.apache.org/jira/browse/HBASE-29544) | Assertion errors in BackupAndRestoreThread are not causing IntegrationTestBackupRestore to fail | Major | integration tests | +| [HBASE-29184](https://issues.apache.org/jira/browse/HBASE-29184) | The snapshot type for disabled table is incorrect when snapshot procedure is enabled | Major | snapshots | +| [HBASE-29493](https://issues.apache.org/jira/browse/HBASE-29493) | TestBucketCacheRefCnt.testInBucketCache fails if block found after eviction | Minor | BucketCache | +| [HBASE-29543](https://issues.apache.org/jira/browse/HBASE-29543) | TestFileChangeWatcher fails with Java 8 due to file modification time granularity issue | Minor | . | +| [HBASE-29507](https://issues.apache.org/jira/browse/HBASE-29507) | IntegrationTestBackupRestore is failing because it cannot restore from backup directory | Major | . | +| [HBASE-28951](https://issues.apache.org/jira/browse/HBASE-28951) | Handle simultaneous WAL splitting to recovered edits by multiple worker | Major | . | +| [HBASE-29503](https://issues.apache.org/jira/browse/HBASE-29503) | IntegrationTestBackupRestore is passing even if an exception occurs in the thread(s) it creates | Major | integration tests | +| [HBASE-29296](https://issues.apache.org/jira/browse/HBASE-29296) | Missing critical snapshot expiration checks | Critical | backup&restore, snapshots | +| [HBASE-29463](https://issues.apache.org/jira/browse/HBASE-29463) | Bidirectional serial replication will block if a region’s last edit before rs crashed was from the peer cluster | Critical | Replication | +| [HBASE-29482](https://issues.apache.org/jira/browse/HBASE-29482) | Bulkload fails with viewfs authentication error | Minor | . | +| [HBASE-29472](https://issues.apache.org/jira/browse/HBASE-29472) | Fix splitting algorithms of RegionSplitter tool | Minor | util | +| [HBASE-29447](https://issues.apache.org/jira/browse/HBASE-29447) | WAL Archives Cause Incremental Backup Failures | Major | backup&restore | +| [HBASE-29474](https://issues.apache.org/jira/browse/HBASE-29474) | RegionSplitter.rollingSplit is broken | Major | . | +| [HBASE-28589](https://issues.apache.org/jira/browse/HBASE-28589) | Server side DoNotRetryException not propagated to client | Critical | IPC/RPC | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29614](https://issues.apache.org/jira/browse/HBASE-29614) | Remove static final field modification in tests around Unsafe | Major | test | +| [HBASE-29591](https://issues.apache.org/jira/browse/HBASE-29591) | Add hadoop 3.4.2 in hadoop check | Major | hadoop3, jenkins, scripts | +| [HBASE-29592](https://issues.apache.org/jira/browse/HBASE-29592) | Add hadoop 3.4.2 in client integration tests | Major | hadoop3, jenkins, scripts | +| [HBASE-29590](https://issues.apache.org/jira/browse/HBASE-29590) | Use hadoop 3.4.2 as default hadooop3 dependency | Major | dependencies, hadoop3 | +| [HBASE-29427](https://issues.apache.org/jira/browse/HBASE-29427) | Merge all commits related to custom tiering into the feature branch | Major | . | +| [HBASE-28467](https://issues.apache.org/jira/browse/HBASE-28467) | Integration of time-based priority caching into cacheOnRead read code paths. | Major | BucketCache | +| [HBASE-28469](https://issues.apache.org/jira/browse/HBASE-28469) | Integration of time-based priority caching into compaction paths. | Major | . | +| [HBASE-28535](https://issues.apache.org/jira/browse/HBASE-28535) | Implement a region server level configuration to enable/disable data-tiering | Major | BucketCache | +| [HBASE-28468](https://issues.apache.org/jira/browse/HBASE-28468) | Integration of time-based priority caching logic into cache evictions. | Major | . | +| [HBASE-28466](https://issues.apache.org/jira/browse/HBASE-28466) | Integration of time-based priority logic of bucket cache in prefetch functionality of HBase. | Major | BucketCache | +| [HBASE-28505](https://issues.apache.org/jira/browse/HBASE-28505) | Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering | Major | . | +| [HBASE-28465](https://issues.apache.org/jira/browse/HBASE-28465) | Implementation of framework for time-based priority bucket-cache. | Major | . | + + +### OTHER: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-23671](https://issues.apache.org/jira/browse/HBASE-23671) | Upgrade to JUnit 5 | Major | Filesystem Integration, test | +| [HBASE-29509](https://issues.apache.org/jira/browse/HBASE-29509) | Bump hbase-thirdparty to 4.1.12 | Major | dependencies, thirdparty | +| [HBASE-29527](https://issues.apache.org/jira/browse/HBASE-29527) | Bump org.bouncycastle:bcpkix-jdk18on from 1.78 to 1.81 | Major | dependabot, dependencies, security | + + ## Release 2.6.3 - Unreleased (as of 2025-07-10) diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000000..a168d77d5600 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +When in the hubspot-2.6 branch, or sub-branches of that branch, use mvn11 instead of mvn, and use the hadoop-3.0 maven profile + +When in the master branch, or sub-branches of that branch, use mvn17 instead of mvn, and do not use the hadoop-3.0 maven profile. + +Always run the spotless:apply maven target after making changes. diff --git a/NOTICE.txt b/NOTICE.txt index b48c5475c962..55b8b5ae2bc5 100755 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Apache HBase -Copyright 2007-2022 The Apache Software Foundation +Copyright 2007-2026 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index 058472e1c070..16c85a260a63 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ under the License. # Getting Start To get started using HBase, the full documentation for this release can be found under the doc/ directory that accompanies this README. Using a browser, open the docs/index.html to view the project home page (or browse https://hbase.apache.org). The hbase '[book](https://hbase.apache.org/book.html)' has a 'quick start' section and is where you should being your exploration of the hbase project. -The latest HBase can be downloaded from the [download page](https://hbase.apache.org/downloads.html). +The latest HBase can be downloaded from the [download page](https://hbase.apache.org/downloads/). We use mailing lists to send notice and discuss. The mailing lists and archives are listed [here](http://hbase.apache.org/mail-lists.html) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6e062b7d1876..b1caf20a942c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -16,6 +16,195 @@ # See the License for the specific language governing permissions and # limitations under the License. --> +# HBASE 2.6.5 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HBASE-29930](https://issues.apache.org/jira/browse/HBASE-29930) | *Major* | **Separate packaging and integration check in nightly job to new jenkins job** + +Introduced a new HBase-Integration-Test jenkins job to run packaging and integration test. + + +--- + +* [HBASE-28985](https://issues.apache.org/jira/browse/HBASE-28985) | *Minor* | **Fix memStoreSize metric for a table and add heap and off-heap size metrics** + +The table-level memStoreSize metric was incorrectly summing data size, heap size, and off-heap size together, while the same metric at the region and region server levels reported only data size. This has been fixed so that memStoreSize now consistently reports data size at all levels. + +Two new metrics have been added at both the table and region levels: +- memStoreHeapSize: on-heap memstore size +- memStoreOffHeapSize: off-heap memstore size + + +--- + +* [HBASE-29877](https://issues.apache.org/jira/browse/HBASE-29877) | *Major* | **Load Test Tool doesn't read with TIMELINE consistency, because it sets replica id explicitly** + +New parameter for HBase LTT "--timeline": Use TIMELINE consistency in read operations. + + +--- + +* [HBASE-29569](https://issues.apache.org/jira/browse/HBASE-29569) | *Minor* | **Implement a built-in TieringValueProvider for parsing the date value from the rowkey** + +This introduces a regex based built-in tiering value provider for parsing dates directly from the rowkey portion of Cells. enabling time-based priority comparison. The implementation utilizes a configurable regular expression and date format to isolate a date substring within the RowKey, returning the parsed epoch time in milliseconds. + +Usage: +This feature is controlled via table-level configuration properties. To utilize this provider, specify the class in the TIERING\_VALUE\_PROVIDER property and define the + +Configuration properties : +TIERING\_KEY\_DATE\_PATTERN: Regex pattern for date extraction. +TIERING\_KEY\_DATE\_FORMAT: Expected date format. +TIERING\_KEY\_DATE\_GROUP: The target regex extract group. + +Example using hbase shell: + +----- +hbase(main):003:0\> alter 'orders', {NAME =\> 'cf1', + CONFIGURATION =\> {'hbase.hstore.datatiering.type' =\> 'CUSTOM', + 'TIERING\_KEY\_DATE\_PATTERN' =\> '\_(\\\\d{4}-\\\\d{2}-\\\\d{2})\_', + 'TIERING\_KEY\_DATE\_FORMAT' =\> 'yyyy-MM-dd', + 'TIERING\_KEY\_DATE\_GROUP' =\> '1', + 'hbase.hstore.datatiering.hot.age.millis' =\> '604800000', + 'hbase.hstore.engine.class' =\> 'org.apache.hadoop.hbase.regionserver.CustomTieredStoreEngine', + 'hbase.hstore.compaction.date.tiered.custom.age.limit.millis' =\> '604800000' + } +} +----- + +Note: Please ensure strict validation of regex patterns and date formats to ensure accurate parsing. + + +--- + +* [HBASE-29837](https://issues.apache.org/jira/browse/HBASE-29837) | *Minor* | **Backport HBASE-27355 Separate meta read requests from master and client to branch-2** + +Dispatch the client's meta read request and the master's meta read request to different processing queues. Introduce a new configuration hbase.ipc.server.metacallqueue.handler.factor to determine the size of the meta request processing queue. + + +--- + +* [HBASE-29141](https://issues.apache.org/jira/browse/HBASE-29141) | *Major* | **Calculate default maxQueueLength call queues correctly** + +The default limit for RPC callqueue length was too high previously. The default was approximately the number of RPC handlers ^ 2 (squared), and is now approximately 10 \* number of RPC handlers. The length can still be overridden with hbase.ipc.server.max.callqueue.length. + + +--- + +* [HBASE-29144](https://issues.apache.org/jira/browse/HBASE-29144) | *Blocker* | **Client request fails for KERBEROS with RpcConnectionRegistry** + +SaslClientAuthenticationProviders and SaslServerAuthenticationProviders are not singleton now. + +As you are free to connect to different HBase clusters which have different authentication configurations from a single process, it does not make sense to let them share the same SaslClientAuthenticationProviders. Now we will create a SaslClientAuthenticationProviders for every Connection instance, even if you pass the same Configuration object when calling ConnectionFactory.createConnection. + +For SaslServerAuthenticationProviders, typically we will only initialize it once in production even if it is not singleton as we will only start a single service(like master or region server) in a process, making it not singleton can also benefit UTs as we will start multiple services when starting a mini cluster. + + +--- + +* [HBASE-29582](https://issues.apache.org/jira/browse/HBASE-29582) | *Major* | **Migrate Jamon pages back to JSP** + +Jamon is almost dead so in this issue, we moved our status page back to JSP, which is also old but a standard in Java EE that not easy to be dead. + +See https://lists.apache.org/thread/0c9to6rxzbf5f2x6t0x20408n15rdvmt for more details. + + +--- + +* [HBASE-29796](https://issues.apache.org/jira/browse/HBASE-29796) | *Minor* | **Allow sleepForRetry replication config to be overridden by replication peers** + +Setting a PeerConfig configuration value beginning with "replication.source" will override the value in the ReplicationSource for only that peer. + +If the default value for "replication.source.sleepforretries" is 1000 and there are two peers, '0' and '1', running "update\_peer\_config '0', CONFIG =\> { "replication.source.sleepforretries" =\> "500" }" will result in '0' using a value of 500 and '1' using the default value of 1000. + + +--- + +* [HBASE-29794](https://issues.apache.org/jira/browse/HBASE-29794) | *Major* | **Only exclude tests when its failure rate above a threshold** + +Now we will only exclude a test from nightly and pre commit build when its failure rate on flakydashboard is above 20% and have been run for at least 10 times. + + +--- + +* [HBASE-29526](https://issues.apache.org/jira/browse/HBASE-29526) | *Major* | **Dynamic configuration not working for coprocessor** + +This change fixed the dynamic configuration for coprocessor. + + +--- + +* [HBASE-29729](https://issues.apache.org/jira/browse/HBASE-29729) | *Minor* | **Add per-region table descriptor hash to RegionServer JMX metrics** + +Adds a tableDescriptorHash metric to region JMX output, exposing a SHA-256 hash of the region's current table descriptor. This enables operators to track which regions have adopted new descriptors during lazy table modifications, supporting progressive canary rollouts without maintaining external state. The hash is computed once at region open and cached for the region's lifetime. + + +--- + +* [HBASE-29740](https://issues.apache.org/jira/browse/HBASE-29740) | *Major* | **Upgrade lz4-java to 1.8.1+** + +Upgrade lz4-java to 1.10.1 for addressing CVE‐2025‐12183 and CVE-2025-66566. + + +--- + +* [HBASE-29695](https://issues.apache.org/jira/browse/HBASE-29695) | *Minor* | **Fix typo in RestoreRequest's "Builder.withOvewrite()" method** + +fixed typo of an internal API by replacing withOvewrite with withOverwrite + + + +# HBASE 2.6.4 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HBASE-29663](https://issues.apache.org/jira/browse/HBASE-29663) | *Major* | **TimeBasedLimiters should support dynamic configuration refresh** + +You can now refresh TimeBasedRateLimiter configurations dynamically with a call to the HBase shell's update\_all\_configs + + +--- + +* [HBASE-29573](https://issues.apache.org/jira/browse/HBASE-29573) | *Minor* | **Fully load QuotaCache instead of reading individual rows on demand** + +Each RegionServer now always caches the full contents of the hbase:quota table. If you have a very large quota table, this could cause excessive memory usage. + + +--- + +* [HBASE-28463](https://issues.apache.org/jira/browse/HBASE-28463) | *Major* | **Time Based Priority for BucketCache** + +This introduces time based priority for blocks in the BucketCache. It's disabled by default. Allows for defining an age threshold at individual column family configuration, whereby blocks older than this configured threshold would be targeted first for eviction. Blocks from column families that don't define the age threshold wouldn't be evaluated by the time based priority, and would only be evicted following the pre-existing LRU eviction logic. + +To enable it, first set the hbase.regionserver.datatiering.enable property to true in the RegionServer configuration. Then, for each table column family where time based priority behaviour is desired, add the following properties to the related column families configurations: +- hbase.hstore.datatiering.type -\> TIME\_RANGE or CUSTOM +- hbase.hstore.datatiering.hot.age.millis -\> A milliseconds age value (defaults to 7 Days or 604800000 milliseconds) +- hbase.hstore.engine.class -\> org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine or org.apache.hadoop.hbase.regionserver.CustomTieredStoreEngine + +The TIME\_RANGE value for hbase.hstore.datatiering.type will rely on cells timestamps for calculating the block age to be compared against the hbase.hstore.datatiering.hot.age.millis threshold age to decide on the block priority. This option requires that org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine be defined as the hbase.hstore.engine.class. This is to enable date tiered compaction, so that data can be placed at separate files, according to the cells timestamps and the age threshold. + +The CUSTOM value for hbase.hstore.datatiering.type allows for defining custom logic to identify the age of cells that should be compared against the threshold age defined in the hbase.hstore.datatiering.hot.age.millis property. This option requires that org.apache.hadoop.hbase.regionserver.CustomTieredStoreEngine be defined as the hbase.hstore.engine.class. This is to enable the custom tiered compaction, so that data can be placed at separate files, according to the custom logic for defining the cell age to be compared against the age threshold. The custom logic for defining cell age should be provided as implementations of the CustomTieredCompactor.TieringValueProvider interface, and should be specified as the value of the hbase.hstore.custom-tiering-value.provider.class. + +Additionally, a built-in implementation of CustomTieredCompactor.TieringValueProvider is provided and set by default when the CUSTOM value for hbase.hstore.datatiering.type is in use. This assumes a custom column qualifier value to contain a long timestamp to be used as the cell age to be compared against the configured age threshold. This column qualifier should be configured as the TIERING\_CELL\_QUALIFIER property in the given column family configuration. + +Note that major compaction needs to be completed on the related tables once the feature is configured properly at the related column families configurations. + + +--- + +* [HBASE-15625](https://issues.apache.org/jira/browse/HBASE-15625) | *Minor* | **Make minimum values configurable and smaller** + +Introduced a new configuration \`hbase.regionserver.free.heap.min.memory.size\`. +This configuration allows users to specify the minimum required amount of free heap memory using a human-readable format (e.g., 512m, 4g). By default, it remains consistent with the previous behavior, reserving 20% of the total heap size as free memory. This new option helps modern deployments with large heap sizes fine-tune memory usage more aggressively for MemStore and block cache configurations. + + + # HBASE 2.6.3 Release Notes These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. diff --git a/build-scripts/prepare_environment.sh b/build-scripts/prepare_environment.sh new file mode 100755 index 000000000000..65842dcd4d17 --- /dev/null +++ b/build-scripts/prepare_environment.sh @@ -0,0 +1,97 @@ +# +# Generates the appropriate environment vars so that we: +# - build against the right version of hadoop, and properly set up maven +# - generate the correct maven version based on the branches +# - upload RPMs with the correct release based on the branch, and to the right yum repo +# +# Since we need to distribute .blazar.yaml to all sub-modules of the project, we define our constants once +# in this script which can be re-used by every .blazar.yaml. +# +set -ex +printenv + +# We base the expected main branch and resulting maven version for clients on the hbase minor version +# The reason for this is hbase re-branches for each minor release (2.4, 2.5, 2.6, etc). At each re-branch +# the histories diverge. So we'll need to create our own fork of each new minor release branch. +# The convention is a fork named "hubspot-$minorVersion", and the maven coordinates "$minorVersion-hubspot-SNAPSHOT" +MINOR_VERSION="2.6" +MAIN_BRANCH="hubspot-${MINOR_VERSION}" + +# +# Validate inputs from blazar +# + +if [ -z "$WORKSPACE" ]; then + echo "Missing env var \$WORKSPACE" + exit 1 +fi +if [ -z "$GIT_BRANCH" ]; then + echo "Missing env var \$GIT_BRANCH" + exit 1 +fi +if [ -z "$BUILD_COMMAND_RC_FILE" ]; then + echo "Missing env var \$BUILD_COMMAND_RC_FILE" + exit 1 +fi + +# +# Extract current hbase version from root pom.xml +# + +# the pom.xml has an invalid xml namespace, so just remove that so xmllint can parse it. +cat $WORKSPACE/pom.xml | sed '2 s/xmlns=".*"//g' > pom.xml.tmp +HBASE_VERSION=$(echo "cat /project/properties/revision/text()" | xmllint --nocdata --shell pom.xml.tmp | sed '1d;$d') +rm pom.xml.tmp + +# sanity check that we've got some that looks right. it wouldn't be the end of the world if we got it wrong, but +# will help avoid confusion. +if [[ ! "$HBASE_VERSION" =~ 2\.[0-9]+\.[0-9]+ ]]; then + echo "Unexpected HBASE_Version extracted from pom.xml. Got $HBASE_VERSION but expected a string like '2.4.3', with 3 numbers separated by decimals, the first number being 2." + exit 1 +fi + +# +# Generate branch-specific env vars +# We are going to generate the maven version and the RPM release here: +# - For the maven version, we need to special case our main branch +# - For RPM, we want our final version to be: +# main branch: {hbase_version}-hs.{build_number}.el6 +# other branches: {hbase_version}-hs~{branch_name}.{build_number}.el6, where branch_name substitutes underscore for non-alpha-numeric characters +# + +echo "Git branch $GIT_BRANCH. Detecting appropriate version override and RPM release." + +RELEASE="hs" + +if [[ "$GIT_BRANCH" = "$MAIN_BRANCH" ]]; then + SET_VERSION="${MINOR_VERSION}-hubspot-SNAPSHOT" + REPO_NAME="AnyLinuxVersion_hs-hbase" +elif [[ "$GIT_BRANCH" != "hubspot" ]]; then + SET_VERSION="${MINOR_VERSION}-${GIT_BRANCH}-SNAPSHOT" + RELEASE="${RELEASE}~${GIT_BRANCH//[^[:alnum:]]/_}" + REPO_NAME="AnyLinuxVersion_hs-hbase-develop" +else + echo "Invalid git branch $GIT_BRANCH" + exit 1 +fi + +RELEASE="${RELEASE}.${BUILD_NUMBER}" +FULL_BUILD_VERSION="${HBASE_VERSION}-${RELEASE}" + +# SET_VERSION is not the most intuitive name, but it's required for set-maven-versions script +write-build-env-var SET_VERSION "$SET_VERSION" +write-build-env-var HBASE_VERSION "$HBASE_VERSION" +write-build-env-var PKG_RELEASE "$RELEASE" +write-build-env-var FULL_BUILD_VERSION "$FULL_BUILD_VERSION" +write-build-env-var REPO_NAME "$REPO_NAME" +# Adding this value as versioninfo.version ensures we have the same value as would normally +# show up in a non-hubspot hbase build. Otherwise due to set-maven-versions we'd end up +# with 2.6-hubspot-SNAPSHOT which is not very useful as a point of reference. +# Another option would be to pass in our FULL_BUILD_VERSION but that might cause some funniness +# with the expectations in VersionInfo.compareVersion(). +write-build-env-var MAVEN_BUILD_ARGS "$MAVEN_BUILD_ARGS -Dversioninfo.version=$HBASE_VERSION" + +echo "Building HBase version $HBASE_VERSION" +echo "Will deploy to nexus with version $SET_VERSION" +echo "Will create rpm with version $FULL_BUILD_VERSION" +echo "Will run maven with extra args $MAVEN_BUILD_ARGS" diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 4bc418017c0a..f22e67d04e83 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -38,7 +38,6 @@ pipeline { OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3' OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3' OUTPUT_DIR_RELATIVE_JDK17_HADOOP3 = 'output-jdk17-hadoop3' - OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS = 'output-jdk17-hadoop3-backwards' PROJECT = 'hbase' PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' @@ -58,9 +57,6 @@ pipeline { ASF_NIGHTLIES = 'https://nightlies.apache.org' ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}" ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}" - // These are dependent on the branch - HADOOP3_VERSIONS = "3.3.5,3.3.6,3.4.0,3.4.1" - HADOOP3_DEFAULT_VERSION = "3.4.1" } parameters { booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. @@ -85,7 +81,7 @@ pipeline { // can't just do a simple echo or the directory won't be created. :( sh '''#!/usr/bin/env bash echo "Make sure we have a directory for downloading dependencies: $(pwd)" -''' + ''' } sh '''#!/usr/bin/env bash set -e @@ -127,77 +123,8 @@ pipeline { } stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh" } - } - stage ('hadoop 2 cache') { - environment { - HADOOP2_VERSION="2.10.2" - } - steps { - // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( - dir('downloads-hadoop-2') { - sh '''#!/usr/bin/env bash - echo "Make sure we have a directory for downloading dependencies: $(pwd)" -''' - } - sh '''#!/usr/bin/env bash - set -e - echo "Ensure we have a copy of Hadoop ${HADOOP2_VERSION}" - "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ - --working-dir "${WORKSPACE}/downloads-hadoop-2" \ - --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ - --verify-tar-gz \ - "${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \ - "hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz" - for stale in $(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | grep -v ${HADOOP2_VERSION}); do - echo "Delete stale hadoop 2 cache ${stale}" - rm -rf $stale - done - ''' - stash name: 'hadoop-2', includes: "hadoop-${HADOOP2_VERSION}-bin.tar.gz" - } - } - stage ('hadoop 3 cache') { - steps { - script { - hadoop3_versions = env.HADOOP3_VERSIONS.split(","); - env.HADOOP3_VERSIONS_REGEX = "[" + hadoop3_versions.join("|") + "]"; - for (hadoop3_version in hadoop3_versions) { - env.HADOOP3_VERSION = hadoop3_version; - echo "env.HADOOP3_VERSION" + env.hadoop3_version; - stage ('Hadoop 3 cache inner stage') { - // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( - dir("downloads-hadoop-${HADOOP3_VERSION}") { - sh '''#!/usr/bin/env bash - echo "Make sure we have a directory for downloading dependencies: $(pwd)" -''' - } //dir - sh '''#!/usr/bin/env bash - set -e - echo "Ensure we have a copy of Hadoop ${HADOOP3_VERSION}" - "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ - --working-dir "${WORKSPACE}/downloads-hadoop-${HADOOP3_VERSION}" \ - --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ - --verify-tar-gz \ - "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \ - "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz" - for stale in $(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | grep -v ${HADOOP3_VERSION}); do - echo "Delete stale hadoop 3 cache ${stale}" - rm -rf $stale - done - ''' - stash name: "hadoop-${HADOOP3_VERSION}", includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz" - script { - if (env.HADOOP3_VERSION == env.HADOOP3_DEFAULT_VERSION) { - // FIXME: we never unstash this, because we run the packaging tests with the version-specific stashes - stash(name: "hadoop-3", includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz") - } //if - } //script - } //stage ('Hadoop 3 cache inner stage') - } //for - } //script - } //steps - } //stage ('hadoop 3 cache') { - } //parallel + } // stage ('yetus install') + } // parallel } //stage ('thirdparty installs') stage ('init health results') { steps { @@ -208,13 +135,6 @@ pipeline { stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match" stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match" stash name: 'jdk17-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/doesn't-match" - script { - for (hadoop3_version in hadoop3_versions) { - // confusing environment vs Groovy variables - stash(name: "jdk17-hadoop3-backwards-result-${hadoop3_version}", allowEmpty: true, includes: "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-${hadoop3_version}/doesn't-match") - } - } - stash name: 'srctarball-result', allowEmpty: true, includes: "output-srctarball/doesn't-match" } } stage ('health checks') { @@ -771,352 +691,6 @@ pipeline { } } } - // If/when we transition to transient runners, we could run every Hadoop check as a matrix job - stage ('yetus jdk17 hadoop3 backwards compatibility checks') { - agent { - node { - label 'hbase' - } - } - environment { - BASEDIR = "${env.WORKSPACE}/component" - TESTS = "${env.DEEP_CHECKS}" - SET_JAVA_HOME = "/usr/lib/jvm/java-17" - // Activates hadoop 3.0 profile in maven runs. - HADOOP_PROFILE = '3.0' - // HADOOP_THREE_VERSION is set in script for loop - TEST_PROFILE = 'runDevTests' - SKIP_ERRORPRONE = true - } - steps { - script { - for (hadoop3_version in hadoop3_versions) { - if (hadoop3_version == env.HADOOP3_DEFAULT_VERSION) { - // We are running the full test suite, no need to run the dev tests too - continue - } - //HADOOP_THREE_VERSION is the environment variable name expected by the nightly shell script - env.HADOOP_THREE_VERSION = hadoop3_version; - env.OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-${env.HADOOP_THREE_VERSION}" - env.OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-${env.HADOOP_THREE_VERSION}" - try { - stage ('yetus jdk17 hadoop3 backwards compatibility checks inner stage') { - // Must do prior to anything else, since if one of them timesout we'll stash the commentfile - sh '''#!/usr/bin/env bash - set -e - rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" - rm -f "${OUTPUT_DIR}/commentfile" - ''' - unstash 'yetus' - dir('component') { - checkout scm - } - sh '''#!/usr/bin/env bash - set -e - rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" - "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" - echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" - ls -lh "${OUTPUT_DIR_RELATIVE}/machine" - ''' - script { - def ret = sh( - returnStatus: true, - script: '''#!/usr/bin/env bash - set -e - declare -i status=0 - if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then - echo "(/) {color:green}+1 jdk17 hadoop ${HADOOP_THREE_VERSION} backward compatibility checks{color}" > "${OUTPUT_DIR}/commentfile" - else - echo "(x) {color:red}-1 jdk17 hadoop ${HADOOP_THREE_VERSION} backward compatibility checks{color}" > "${OUTPUT_DIR}/commentfile" - status=1 - fi - echo "-- For more information [see jdk17 report|${BUILD_URL}JDK17_20Nightly_20Build_20Report_20_28Hadoop3_29/]" >> "${OUTPUT_DIR}/commentfile" - exit "${status}" - ''' - ) - if (ret != 0) { - // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of - // test output. See HBASE-26339 for more details. - currentBuild.result = 'UNSTABLE' - } - } //script - } //stage ('yetus jdk17 hadoop3 backwards compatibility checks inner stage') { - } //try - finally { - stash name: "jdk17-hadoop3-backwards-result-${HADOOP_THREE_VERSION}", includes: "${OUTPUT_DIR_RELATIVE}/commentfile" - junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true - // zip surefire reports. - sh '''#!/bin/bash -e - if [ ! -f "${OUTPUT_DIR}/commentfile" ]; then - echo "(x) {color:red}-1 jdk17 hadoop ${HADOOP_THREE_VERSION} backward compatibility checks{color}" >"${OUTPUT_DIR}/commentfile" - echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" - fi - if [ -d "${OUTPUT_DIR}/archiver" ]; then - count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l) - if [[ 0 -ne ${count} ]]; then - echo "zipping ${count} archived files" - zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver" - else - echo "No archived files, skipping compressing." - fi - else - echo "No archiver directory, skipping compressing." - fi - ''' - sshPublisher(publishers: [ - sshPublisherDesc(configName: 'Nightlies', - transfers: [ - sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", - sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip" - ) - ] - ) - ]) - // remove the big test logs zip file, store the nightlies url in test_logs.html - sh '''#!/bin/bash -e - if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then - echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" - rm -rf "${OUTPUT_DIR}/test_logs.zip" - python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" - else - echo "No test_logs.zip, skipping" - fi - ''' - // Has to be relative to WORKSPACE. - archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" - archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" - publishHTML target: [ - allowMissing : true, - keepAll : true, - alwaysLinkToLastBuild: true, - // Has to be relative to WORKSPACE. - reportDir : "${env.OUTPUT_DIR_RELATIVE}", - reportFiles : 'console-report.html', - reportName : "JDK17 Nightly Build Report (Hadoop ${HADOOP_THREE_VERSION} backwards compatibility)" - ] - } //finally - } // for - } //script - } //steps - } //stage ('yetus jdk17 hadoop3 backwards compatibility checks') - - // This is meant to mimic what a release manager will do to create RCs. - // See http://hbase.apache.org/book.html#maven.release - // TODO (HBASE-23870): replace this with invocation of the release tool - stage ('packaging and integration') { - agent { - node { - label 'hbase' - } - } - environment { - BASEDIR = "${env.WORKSPACE}/component" - BRANCH = "${env.BRANCH_NAME}" - } - steps { - dir('component') { - checkout scm - } - sh '''#!/bin/bash -e - echo "Setting up directories" - rm -rf "output-srctarball" && mkdir "output-srctarball" - rm -rf "output-integration" && mkdir "output-integration" "output-integration/hadoop-2" "output-integration/hadoop-3" "output-integration/hadoop-3-shaded" - rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball" - rm -rf "hbase-install" && mkdir "hbase-install" - rm -rf "hbase-client" && mkdir "hbase-client" - rm -rf "hbase-hadoop3-install" - rm -rf "hbase-hadoop3-client" - rm -rf "hadoop-2" && mkdir "hadoop-2" - rm -rf "hadoop-3" && mkdir "hadoop-3" - rm -rf ".m2-for-repo" && mkdir ".m2-for-repo" - rm -rf ".m2-for-src" && mkdir ".m2-for-src" - # remove old hadoop tarballs in workspace - rm -rf hadoop-2*.tar.gz - rm -rf hadoop-3*.tar.gz - rm -f "output-integration/commentfile" - ''' - sh '''#!/usr/bin/env bash - set -e - rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" - "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" - echo "got the following saved stats in 'output-srctarball/machine'" - ls -lh "output-srctarball/machine" - ''' - sh '''#!/bin/bash -e - echo "Checking the steps for an RM to make a source artifact, then a binary artifact." - docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" . - docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ - -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" --workdir=/hbase hbase-integration-test \ - "component/dev-support/hbase_nightly_source-artifact.sh" \ - --intermediate-file-dir output-srctarball \ - --unpack-temp-dir unpacked_src_tarball \ - --maven-m2-initial .m2-for-repo \ - --maven-m2-src-build .m2-for-src \ - --clean-source-checkout \ - component - if [ $? -eq 0 ]; then - echo '(/) {color:green}+1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile - else - echo '(x) {color:red}-1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile - exit 1 - fi - ''' - echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'" - sh '''#!/bin/bash -e - if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v hadoop3 | wc -l) ]; then - echo '(x) {color:red}-1 testing binary artifact{color}\n-- source tarball did not produce the expected binaries.' >>output-srctarball/commentfile - exit 1 - fi - install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v client-bin | grep -v hadoop3) - tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install" - client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-client-bin.tar.gz | grep -v hadoop3) - tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client" - if [ 2 -eq $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz | wc -l) ]; then - echo "hadoop3 artifacts available, unpacking the hbase hadoop3 bin tarball into 'hbase-hadoop3-install' and the client hadoop3 tarball into 'hbase-hadoop3-client'" - mkdir hbase-hadoop3-install - mkdir hbase-hadoop3-client - hadoop3_install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz | grep -v client-bin) - tar --strip-component=1 -xzf "${hadoop3_install_artifact}" -C "hbase-hadoop3-install" - hadoop3_client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-client-bin.tar.gz) - tar --strip-component=1 -xzf "${hadoop3_client_artifact}" -C "hbase-hadoop3-client" - fi - ''' - unstash 'hadoop-2' - sh '''#!/bin/bash -xe - if [[ "${BRANCH}" == *"branch-2"* ]]; then - echo "Attempting to use run an instance on top of Hadoop 2." - artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1) - tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2" - docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" . - docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ - -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-8" --workdir=/hbase hbase-integration-test \ - component/dev-support/hbase_nightly_pseudo-distributed-test.sh \ - --single-process \ - --working-dir output-integration/hadoop-2 \ - --hbase-client-install "hbase-client" \ - hbase-install \ - hadoop-2/bin/hadoop \ - hadoop-2/share/hadoop/yarn/timelineservice \ - hadoop-2/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ - hadoop-2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ - hadoop-2/bin/mapred \ - >output-integration/hadoop-2.log 2>&1 - if [ $? -ne 0 ]; then - echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile - exit 2 - fi - echo "(/) {color:green}+1 client integration test for HBase 2 {color}" >output-integration/commentfile - else - echo "Skipping to run against Hadoop 2 for branch ${BRANCH}" - fi - ''' - script { - for (hadoop3_version in hadoop3_versions) { - env.HADOOP3_VERSION = hadoop3_version; - echo "env.HADOOP3_VERSION" + env.hadoop3_version; - stage ("packaging and integration Hadoop 3 inner stage ") { - unstash "hadoop-" + env.HADOOP3_VERSION - sh '''#!/bin/bash -e - echo "Attempting to use run an instance on top of Hadoop ${HADOOP3_VERSION}." - # Clean up any previous tested Hadoop3 files before unpacking the current one - rm -rf hadoop-3/* - # Create working dir - rm -rf "output-integration/hadoop-${HADOOP3_VERSION}" && mkdir "output-integration/hadoop-${HADOOP3_VERSION}" - rm -rf "output-integration/hadoop-${HADOOP3_VERSION}-shaded" && mkdir "output-integration/hadoop-${HADOOP3_VERSION}-shaded" - artifact=$(ls -1 "${WORKSPACE}"/hadoop-${HADOOP3_VERSION}-bin.tar.gz | head -n 1) - tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3" - # we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471 - ${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3 - hbase_install_dir="hbase-install" - hbase_client_dir="hbase-client" - if [ -d "hbase-hadoop3-install" ]; then - echo "run hadoop3 client integration test against hbase hadoop3 binaries" - hbase_install_dir="hbase-hadoop3-install" - hbase_client_dir="hbase-hadoop3-client" - fi - docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" . - docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ - -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \ - -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \ - --workdir=/hbase hbase-integration-test \ - component/dev-support/hbase_nightly_pseudo-distributed-test.sh \ - --single-process \ - --working-dir output-integration/hadoop-${HADOOP3_VERSION} \ - --hbase-client-install ${hbase_client_dir} \ - ${hbase_install_dir} \ - hadoop-3/bin/hadoop \ - hadoop-3/share/hadoop/yarn/timelineservice \ - hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ - hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ - hadoop-3/bin/mapred \ - >output-integration/hadoop-${HADOOP3_VERSION}.log 2>&1 - if [ $? -ne 0 ]; then - echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop ${HADOOP3_VERSION}. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-${HADOOP3_VERSION}.log]. (note that this means we didn't check the Hadoop ${HADOOP3_VERSION} shaded client)" >> output-integration/commentfile - exit 2 - fi - echo "Attempting to use run an instance on top of Hadoop ${HADOOP3_VERSION}, relying on the Hadoop client artifacts for the example client program." - docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ - -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \ - -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \ - --workdir=/hbase hbase-integration-test \ - component/dev-support/hbase_nightly_pseudo-distributed-test.sh \ - --single-process \ - --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \ - --working-dir output-integration/hadoop-${HADOOP3_VERSION}-shaded \ - --hbase-client-install ${hbase_client_dir} \ - ${hbase_install_dir} \ - hadoop-3/bin/hadoop \ - hadoop-3/share/hadoop/yarn/timelineservice \ - hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ - hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ - hadoop-3/bin/mapred \ - >output-integration/hadoop-${HADOOP3_VERSION}-shaded.log 2>&1 - if [ $? -ne 0 ]; then - echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop ${HADOOP3_VERSION} using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-${HADOOP3_VERSION}-shaded.log]." >> output-integration/commentfile - exit 2 - fi - echo "(/) {color:green}+1 client integration test for ${HADOOP3_VERSION} {color}" >> output-integration/commentfile - ''' - } //stage ("packaging and integration Hadoop 3 inner stage ") - } //for - } // script - } //steps - post { - always { - sh '''#!/bin/bash -e - if [ ! -f "output-integration/commentfile" ]; then - echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile - echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile - fi - ''' - stash name: 'srctarball-result', includes: "output-srctarball/commentfile,output-integration/commentfile" - sshPublisher(publishers: [ - sshPublisherDesc(configName: 'Nightlies', - transfers: [ - sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", - sourceFiles: "output-srctarball/hbase-src.tar.gz" - ) - ] - ) - ]) - // remove the big src tarball, store the nightlies url in hbase-src.html - sh '''#!/bin/bash -e - SRC_TAR="${WORKSPACE}/output-srctarball/hbase-src.tar.gz" - if [ -f "${SRC_TAR}" ]; then - echo "Remove ${SRC_TAR} for saving space" - rm -rf "${SRC_TAR}" - python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html" - else - echo "No hbase-src.tar.gz, skipping" - fi - ''' - archiveArtifacts artifacts: 'output-srctarball/*' - archiveArtifacts artifacts: 'output-srctarball/**/*' - archiveArtifacts artifacts: 'output-integration/*' - archiveArtifacts artifacts: 'output-integration/**/*' - } //always - } //post - } //stage packaging } // parallel } //stage:_health checks } //stages @@ -1133,32 +707,18 @@ pipeline { rm -rf ${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3} rm -rf ${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3} rm -rf ${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3} - rm -rf ${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-* - rm -rf output-srctarball - rm -rf output-integration ''' unstash 'general-result' unstash 'jdk8-hadoop2-result' unstash 'jdk8-hadoop3-result' unstash 'jdk11-hadoop3-result' unstash 'jdk17-hadoop3-result' - unstash 'srctarball-result' def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/commentfile"] - for (hadoop3_version in hadoop3_versions) { - if (hadoop3_version == env.HADOOP3_DEFAULT_VERSION) { - // We haven't run these tests - continue - } - unstash("jdk17-hadoop3-backwards-result-${hadoop3_version}") - results.add("${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-${hadoop3_version}/commentfile") - } - results.add('output-srctarball/commentfile') - results.add('output-integration/commentfile') echo env.BRANCH_NAME echo env.BUILD_URL echo currentBuild.result diff --git a/dev-support/blanks-eol-ignore.txt b/dev-support/blanks-eol-ignore.txt index 6912be308371..51edc75be9d6 100644 --- a/dev-support/blanks-eol-ignore.txt +++ b/dev-support/blanks-eol-ignore.txt @@ -20,5 +20,5 @@ .*/gen-cpp/.* .*/gen-perl/.* .*/gen-php/.* -.*/gen-py/.* +.*/gen_py/.* .*/gen-rb/.* diff --git a/dev-support/blanks-tabs-ignore.txt b/dev-support/blanks-tabs-ignore.txt index 49185487846e..54f35708c4d8 100644 --- a/dev-support/blanks-tabs-ignore.txt +++ b/dev-support/blanks-tabs-ignore.txt @@ -20,7 +20,7 @@ .*/gen-cpp/.* .*/gen-perl/.* .*/gen-php/.* -.*/gen-py/.* +.*/gen_py/.* .*/gen-rb/.* # we have tabs in asciidoc, not sure whether it is OK to replace them with spaces src/main/asciidoc/.* diff --git a/dev-support/flaky-tests/report-flakies.py b/dev-support/flaky-tests/report-flakies.py index 16096e3344a5..33e9a12f0d5e 100755 --- a/dev-support/flaky-tests/report-flakies.py +++ b/dev-support/flaky-tests/report-flakies.py @@ -54,6 +54,11 @@ parser.add_argument('--is-yetus', metavar='True/False', action='append', choices=['True', 'False'], help='True, if build is yetus style i.e. look for maven output in artifacts; ' 'False, if maven output is in /consoleText itself.') +parser.add_argument('--excludes-threshold-flakiness', metavar='n', type=float, default=20.0, + required=False, help='Flakiness threshold for adding a test to excludes file') +parser.add_argument('--excludes-threshold-runs', metavar='n', type=int, default=10, + required=False, + help='The times of a test should run before it can be added to excludes file') parser.add_argument( "--mvn", action="store_true", help="Writes two strings for including/excluding these flaky tests using maven flags. These " @@ -149,7 +154,6 @@ def expand_multi_config_projects(cli_args): 'excludes': excluded_builds, 'is_yetus': is_yetus}) return final_expanded_urls - # Set of timeout/failed tests across all given urls. all_timeout_tests = set() all_failed_tests = set() @@ -160,6 +164,8 @@ def expand_multi_config_projects(cli_args): # Contains { : [run_ids] } # Used for common min/max build ids when generating sparklines. url_to_build_ids = OrderedDict() +all_flaky_results = {} + # Iterates over each url, gets test results and prints flaky tests. expanded_urls = expand_multi_config_projects(args) @@ -205,36 +211,46 @@ def expand_multi_config_projects(cli_args): bad_tests.update(failed_tests.union(hanging_tests)) # For each bad test, get build ids where it ran, timed out, failed or hanged. - test_to_build_ids = {key : {'all' : set(), 'timeout': set(), 'failed': set(), - 'hanging' : set(), 'bad_count' : 0} + test_to_build_ids = {key: {'all': set(), 'timeout': set(), 'failed': set(), + 'hanging': set(), 'bad_count': 0} for key in bad_tests} + for build in build_id_to_results: [all_tests, failed_tests, timeout_tests, hanging_tests] = build_id_to_results[build] - for bad_test in test_to_build_ids: + for bad_test, test_result in test_to_build_ids.items(): is_bad = False if all_tests.issuperset([bad_test]): - test_to_build_ids[bad_test]["all"].add(build) + test_result["all"].add(build) if timeout_tests.issuperset([bad_test]): - test_to_build_ids[bad_test]['timeout'].add(build) + test_result['timeout'].add(build) is_bad = True if failed_tests.issuperset([bad_test]): - test_to_build_ids[bad_test]['failed'].add(build) + test_result['failed'].add(build) is_bad = True if hanging_tests.issuperset([bad_test]): - test_to_build_ids[bad_test]['hanging'].add(build) + test_result['hanging'].add(build) is_bad = True if is_bad: - test_to_build_ids[bad_test]['bad_count'] += 1 + test_result['bad_count'] += 1 # Calculate flakyness % and successful builds for each test. Also sort build ids. - for bad_test in test_to_build_ids: - test_result = test_to_build_ids[bad_test] + for bad_test, test_result in test_to_build_ids.items(): test_result['flakyness'] = test_result['bad_count'] * 100.0 / len(test_result['all']) test_result['success'] = (test_result['all'].difference( test_result['failed'].union(test_result['hanging']))) for key in ['all', 'timeout', 'failed', 'hanging', 'success']: test_result[key] = sorted(test_result[key]) - + # record flaky test result + # record the one with more runs, or greater flakiness if runs are equal + if bad_test not in all_flaky_results: + all_flaky_results[bad_test] = {'runs': len(test_result['all']), + 'flakyness': test_result['flakyness']} + elif all_flaky_results[bad_test]['runs'] < len(test_result['all']): + all_flaky_results[bad_test] = {'runs': len(test_result['all']), + 'flakyness': test_result['flakyness']} + elif all_flaky_results[bad_test]['runs'] == len(test_result['all']) and \ + all_flaky_results[bad_test]['flakyness'] < test_result['flakyness']: + all_flaky_results[bad_test]['flakyness'] = test_result['flakyness'] # Sort tests in descending order by flakyness. sorted_test_to_build_ids = OrderedDict( @@ -260,14 +276,21 @@ def expand_multi_config_projects(cli_args): print("Builds without any test runs: {}".format(build_ids_without_tests_run)) print("") - all_bad_tests = all_hanging_tests.union(all_failed_tests) if args.mvn: includes = ",".join(all_bad_tests) with open(output_dir + "/includes", "w") as inc_file: inc_file.write(includes) - excludes = ["**/{0}.java".format(bad_test) for bad_test in all_bad_tests] + excludes = [] + for bad_test in all_bad_tests: + if bad_test not in all_flaky_results: + print(f"No flaky record found for {bad_test}") + continue + test_result = all_flaky_results[bad_test] + if test_result['flakyness'] > args.excludes_threshold_flakiness and \ + test_result['runs'] >= args.excludes_threshold_runs: + excludes.append(f"**/{bad_test}.java") with open(output_dir + "/excludes", "w") as exc_file: exc_file.write(",".join(excludes)) @@ -283,5 +306,5 @@ def expand_multi_config_projects(cli_args): with open(output_dir + "/dashboard.html", "w") as f: datetime = time.strftime("%m/%d/%Y %H:%M:%S") - f.write(template.render(datetime=datetime, bad_tests_count=len(all_bad_tests), + f.write(template.render(datetime=datetime, bad_tests_count=len(bad_tests), results=url_to_bad_test_results, build_ids=url_to_build_ids)) diff --git a/dev-support/gh_hide_old_comments.sh b/dev-support/gh_hide_old_comments.sh index 61217cfa241b..abba55a39ad2 100755 --- a/dev-support/gh_hide_old_comments.sh +++ b/dev-support/gh_hide_old_comments.sh @@ -40,25 +40,79 @@ declare CURL="${CURL:-curl}" function fetch_comments { local pr="$1" local comments_file + local page_file + local headers_file local -a curl_args - curl_args=( - --fail - "${GITHUB_AUTH[@]}" - --header 'Accept: application/vnd.github+json' - --header 'X-GitHub-Api-Version: 2022-11-28' - --request GET - --url "${GITHUB_API_URL}/repos/${REPO}/issues/${pr}/comments?per_page=500" - ) - if [ "${DEBUG}" = true ] ; then - curl_args+=(--verbose) - else - curl_args+=(--silent) - fi + local page=1 + local next_url comments_file="$(mktemp "comments_${pr}" 2>/dev/null || mktemp -t "comments_${pr}.XXXXXXXXXX")" || \ { >&2 echo 'cannot create temp file'; exit 1 ;} - "${CURL}" "${curl_args[@]}" > "${comments_file}" + page_file="$(mktemp "page_${pr}" 2>/dev/null || mktemp -t "page_${pr}.XXXXXXXXXX")" || \ + { >&2 echo 'cannot create temp file'; exit 1 ;} + headers_file="$(mktemp "headers_${pr}" 2>/dev/null || mktemp -t "headers_${pr}.XXXXXXXXXX")" || \ + { >&2 echo 'cannot create temp file'; exit 1 ;} + + # cleanup temp files on error + trap 'rm -f "${page_file}" "${headers_file}"; exit 1' ERR + + next_url="${GITHUB_API_URL}/repos/${REPO}/issues/${pr}/comments?per_page=100" + + # start with empty JSON array + echo '[]' > "${comments_file}" + + while [ -n "${next_url}" ] ; do + curl_args=( + --fail + --max-time 30 + "${GITHUB_AUTH[@]}" + --header 'Accept: application/vnd.github+json' + --header 'X-GitHub-Api-Version: 2022-11-28' + --dump-header "${headers_file}" + --request GET + --url "${next_url}" + ) + if [ "${DEBUG}" = true ] ; then + curl_args+=(--verbose) + >&2 echo "Fetching page ${page}: ${next_url}" + else + curl_args+=(--silent) + fi + + if ! "${CURL}" "${curl_args[@]}" > "${page_file}"; then + >&2 echo "Failed to fetch page ${page}: ${next_url}" + rm -f "${page_file}" "${headers_file}" + exit 1 + fi + + if [ "${DEBUG}" = 'true' ] ; then + >&2 echo "Page ${page} returned $(jq length "${page_file}") comments" + fi + + # merge this page into the accumulated results + if ! jq -s '.[0] + .[1]' "${comments_file}" "${page_file}" > "${comments_file}.tmp"; then + >&2 echo "Failed to merge comments from page ${page}" + rm -f "${page_file}" "${headers_file}" "${comments_file}.tmp" + exit 1 + fi + mv "${comments_file}.tmp" "${comments_file}" + + # check for next page in Link header + # Link header format: ; rel="next", ; rel="last" + # Extract URL associated with rel="next" regardless of position + next_url="" + if grep -qi '^link:' "${headers_file}" ; then + next_url=$(grep -i '^link:' "${headers_file}" | tr ',' '\n' | grep 'rel="next"' | sed -n 's/.*<\([^>]*\)>.*/\1/p' || true) + fi + + page=$((page + 1)) + done + + rm -f "${page_file}" "${headers_file}" + trap - ERR + if [ "${DEBUG}" = 'true' ] ; then + >&2 echo "Total comments fetched: $(jq length "${comments_file}")" >&2 cat "${comments_file}" fi echo "${comments_file}" @@ -104,11 +158,16 @@ function identify_most_recent_build_number { local pr="$1" local comments_file="$2" local jq_filter + local url_pattern="${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/" + # GitHub Actions URLs don't have /job/ in them + if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then + url_pattern="${JOB_NAME}/PR-${pr}/(?[0-9]+)/" + fi read -r -d '' jq_filter << EOF || : .[] \ | select(.user.id == ${BUILD_BOT_USER_ID}) \ | .body \ -| capture("${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/") \ +| capture("${url_pattern}") \ | .buildnum EOF @@ -122,10 +181,15 @@ function identify_old_comment_ids { local comments_file="$2" local most_recent_build_number="$3" local jq_filter + local url_pattern="${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/" + # GitHub Actions URLs don't have /job/ in them + if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then + url_pattern="${JOB_NAME}/PR-${pr}/(?[0-9]+)/" + fi read -r -d '' jq_filter << EOF || : .[] \ | select(.user.id == ${BUILD_BOT_USER_ID}) \ -| { node_id, buildnum: (.body | capture("${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/") | .buildnum | tonumber) } \ +| { node_id, buildnum: (.body | capture("${url_pattern}") | .buildnum | tonumber) } \ | select(.buildnum < (${most_recent_build_number} | tonumber)) \ | .node_id EOF diff --git a/dev-support/hadoop3-backwards-compatibility-check.Jenkinsfile b/dev-support/hadoop3-backwards-compatibility-check.Jenkinsfile new file mode 100644 index 000000000000..8e8eff8ded30 --- /dev/null +++ b/dev-support/hadoop3-backwards-compatibility-check.Jenkinsfile @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Jenkinsfile for Hadoop3 Backwards Compatibility Checks +// Uses matrix job to parallelize checks across different Hadoop3 versions + +pipeline { + agent { + node { + label 'hbase' + } + } + triggers { + pollSCM('H H */2 * *') + } + options { + buildDiscarder(logRotator(numToKeepStr: '20')) + timeout (time: 8, unit: 'HOURS') + timestamps() + skipDefaultCheckout() + disableConcurrentBuilds() + } + environment { + YETUS_RELEASE = '0.15.0' + HADOOP_VERSIONS = "3.2.4,3.3.5,3.3.6,3.4.0,3.4.1,3.4.2" + } + parameters { + booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. + + Should only be used manually when e.g. there is some non-work-aroundable issue in yetus we are checking a fix for.''') + booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') + } + stages { + stage ('scm-checkout') { + steps { + dir('component') { + checkout scm + } + } + } + stage ('thirdparty installs') { + parallel { + stage ('yetus install') { + steps { + dir('downloads-yetus') { + sh '''#!/usr/bin/env bash + echo "Make sure we have a directory for downloading dependencies: $(pwd)" + ''' + } + sh '''#!/usr/bin/env bash + set -e + echo "Ensure we have a copy of Apache Yetus." + if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then + YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" + echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'" + if ! "${YETUS_DIR}/bin/test-patch" --version >/dev/null 2>&1 ; then + rm -rf "${YETUS_DIR}" + "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --working-dir "${WORKSPACE}/downloads-yetus" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \ + "yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz" + mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz + else + echo "Reusing cached install of Apache Yetus version ${YETUS_RELEASE}." + fi + else + YETUS_DIR="${WORKSPACE}/yetus-git" + rm -rf "${YETUS_DIR}" + echo "downloading from github" + curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz + fi + if [ ! -d "${YETUS_DIR}" ]; then + echo "unpacking yetus into '${YETUS_DIR}'" + mkdir -p "${YETUS_DIR}" + gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1 + fi + ''' + stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*" + } + } + } + } + stage ('backwards compatibility checks') { + matrix { + axes { + axis { + name 'HADOOP3_VERSION' + values '3.2.4', '3.3.5', '3.3.6', '3.4.0', '3.4.1', '3.4.2' + } + } + agent { + node { + label 'hbase' + } + } + when { + expression { + if (HADOOP3_VERSION == '3.2.4') { + // only branch-2.5 need to run against hadoop 3.2.4, here we also includes + // HBASE-XXXXX-branch-2.5 feature branch + return env.BRANCH_NAME.contains('branch-2.5') + } + return true + } + } + environment { + PROJECT = 'hbase' + BASEDIR = "${WORKSPACE}/component" + PERSONALITY_FILE = "${BASEDIR}/dev-support/hbase-personality.sh" + TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop' + EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes" + ASF_NIGHTLIES = 'https://nightlies.apache.org' + ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}" + ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}" + TESTS = 'compile,htmlout,javac,maven,mvninstall,shadedjars,unit' + SET_JAVA_HOME = "/usr/lib/jvm/java-17" + HADOOP_PROFILE = '3.0' + TEST_PROFILE = 'runDevTests' + SKIP_ERRORPRONE = true + OUTPUT_DIR_RELATIVE = "output-jdk17-hadoop3-backwards-${HADOOP3_VERSION}" + OUTPUT_DIR = "${WORKSPACE}/${OUTPUT_DIR_RELATIVE}" + AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc' + BLANKS_EOL_IGNORE_FILE = 'dev-support/blanks-eol-ignore.txt' + BLANKS_TABS_IGNORE_FILE = 'dev-support/blanks-tabs-ignore.txt' + // output from surefire; sadly the archive function in yetus only works on file names. + ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump' + } + stages { + stage ('run checks') { + steps { + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" + rm -f "${OUTPUT_DIR}/commentfile" + ''' + unstash 'yetus' + dir('component') { + checkout scm + } + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" + echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" + ls -lh "${OUTPUT_DIR_RELATIVE}/machine" + ''' + script { + def ret = sh( + returnStatus: true, + script: '''#!/usr/bin/env bash + set -e + declare -i status=0 + if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then + echo "(/) {color:green}+1 jdk17 hadoop ${HADOOP3_VERSION} backward compatibility checks{color}" > "${OUTPUT_DIR}/commentfile" + else + echo "(x) {color:red}-1 jdk17 hadoop ${HADOOP3_VERSION} backward compatibility checks{color}" > "${OUTPUT_DIR}/commentfile" + status=1 + fi + echo "-- For more information [see jdk17 report|${BUILD_URL}console]" >> "${OUTPUT_DIR}/commentfile" + exit "${status}" + ''' + ) + if (ret != 0) { + currentBuild.result = 'UNSTABLE' + } + } + } + } + } + post { + always { + script { + stash name: "jdk17-hadoop3-backwards-result-${HADOOP3_VERSION}", includes: "${OUTPUT_DIR_RELATIVE}/commentfile" + junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true + // zip surefire reports. + sh '''#!/bin/bash -e + if [ ! -f "${OUTPUT_DIR}/commentfile" ]; then + echo "(x) {color:red}-1 jdk17 hadoop ${HADOOP3_VERSION} backward compatibility checks{color}" >"${OUTPUT_DIR}/commentfile" + echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" + fi + if [ -d "${OUTPUT_DIR}/archiver" ]; then + count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l) + if [[ 0 -ne ${count} ]]; then + echo "zipping ${count} archived files" + zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver" + else + echo "No archived files, skipping compressing." + fi + else + echo "No archiver directory, skipping compressing." + fi + ''' + def logFile = "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip" + if (fileExists(logFile)) { + sshPublisher(publishers: [ + sshPublisherDesc(configName: 'Nightlies', + transfers: [ + sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", + sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip" + ) + ] + ) + ]) + sh '''#!/bin/bash -e + echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" + rm -rf "${OUTPUT_DIR}/test_logs.zip" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + ''' + } + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" + publishHTML target: [ + allowMissing: true, + keepAll: true, + alwaysLinkToLastBuild: true, + reportDir: "${env.OUTPUT_DIR_RELATIVE}", + reportFiles: 'console-report.html', + reportName: "JDK17 Nightly Build Report (Hadoop ${HADOOP3_VERSION} backwards compatibility)" + ] + } // script + } // always + } // post + } // matrix + } // stage ('backwards compatibility checks') + } // stages + post { + always { + script { + sh "printenv" + // wipe out all the output directories before unstashing + sh''' + echo "Clean up result directories" + rm -rf output-jdk17-hadoop3-backwards-* + ''' + def results = [] + for (hadoopVersion in getHadoopVersions(env.HADOOP_VERSIONS)) { + try { + unstash "jdk17-hadoop3-backwards-result-${hadoopVersion}" + results.add("output-jdk17-hadoop3-backwards-${hadoopVersion}/commentfile") + } catch (e) { + echo "unstash ${hadoopVersion} failed, ignore" + } + } + try { + def comment = "Results for branch ${env.BRANCH_NAME}\n" + comment += "\t[build ${currentBuild.displayName} on builds.a.o|${env.BUILD_URL}]: " + if (currentBuild.result == null || currentBuild.result == "SUCCESS") { + comment += "(/) *{color:green}+1 overall{color}*\n" + } else { + comment += "(x) *{color:red}-1 overall{color}*\n" + } + comment += "----\n" + comment += "Backwards compatibility checks:\n" + comment += results.collect { fileExists(file: it) ? readFile(file: it) : "" }.join("\n\n") + + echo "[INFO] Comment:" + echo comment + + def jiras = getJirasToComment(env.BRANCH_NAME, []) + if (jiras.isEmpty()) { + echo "[DEBUG] non-feature branch, checking change messages for jira keys." + jiras = getJirasToCommentFromChangesets(currentBuild) + } + jiras.each { currentIssue -> + jiraComment issueKey: currentIssue, body: comment + } + } catch (Exception exception) { + echo "Got exception: ${exception}" + echo " ${exception.getStackTrace()}" + } + } + } + } +} + +@NonCPS +List getHadoopVersions(String versions) { + return versions.split(',').collect { it.trim() }.findAll { it } as String[] +} + +import org.jenkinsci.plugins.workflow.support.steps.build.RunWrapper +@NonCPS +List getJirasToCommentFromChangesets(RunWrapper thisBuild) { + def seenJiras = [] + thisBuild.changeSets.each { cs -> + cs.getItems().each { change -> + CharSequence msg = change.msg + echo "change: ${change}" + echo " ${msg}" + echo " ${change.commitId}" + echo " ${change.author}" + seenJiras = getJirasToComment(msg, seenJiras) + } + } + return seenJiras +} + +@NonCPS +List getJirasToComment(CharSequence source, List seen) { + source.eachMatch("HBASE-[0-9]+") { currentIssue -> + echo "[DEBUG] found jira key: ${currentIssue}" + if (currentIssue in seen) { + echo "[DEBUG] already commented on ${currentIssue}." + } else { + echo "[INFO] commenting on ${currentIssue}." + seen << currentIssue + } + } + return seen +} diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 46f08276c651..b5bd436064de 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -330,6 +330,9 @@ function personality_file_tests ## @audience private ## @stability evolving ## @param name of variable to set with maven arguments +# NOTE: INCLUDE_TESTS_URL uses -Dtest= which conflicts with pom.xml patterns. +# Do not use INCLUDE_TESTS_URL with profiles that define their own patterns +# (e.g., runLargeTests-wave1, runLargeTests-wave2, runLargeTests-wave3). function get_include_exclude_tests_arg { local __resultvar=$1 @@ -612,17 +615,17 @@ function hadoopcheck_rebuild # TODO remove this on non 2.5 branches ? yetus_info "Setting Hadoop 3 versions to test based on branch-2.5 rules" if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop3_versions="3.2.4 3.3.6 3.4.0" + hbase_hadoop3_versions="3.2.4 3.3.6 3.4.2" else - hbase_hadoop3_versions="3.2.3 3.2.4 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6 3.4.0" + hbase_hadoop3_versions="3.2.3 3.2.4 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6 3.4.0 3.4.1 3.4.2" fi else yetus_info "Setting Hadoop 3 versions to test based on branch-2.6+/master/feature branch rules" # Isn't runnung these tests with the default Hadoop version redundant ? if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop3_versions="3.3.6 3.4.0" + hbase_hadoop3_versions="3.3.6 3.4.2" else - hbase_hadoop3_versions="3.3.5 3.3.6 3.4.0" + hbase_hadoop3_versions="3.3.5 3.3.6 3.4.0 3.4.1 3.4.2" fi fi @@ -852,6 +855,7 @@ function spotless_rebuild { local repostatus=$1 local logfile="${PATCH_DIR}/${repostatus}-spotless.txt" + local linecommentsfile="${PATCH_DIR}/${repostatus}-spotless-linecomments.txt" if ! verify_needed_test spotless; then return 0 @@ -869,12 +873,27 @@ function spotless_rebuild count=$(${GREP} -c '\[ERROR\]' "${logfile}") if [[ ${count} -gt 0 ]]; then - add_vote_table -1 spotless "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix." - add_footer_table spotless "@@BASE@@/${repostatus}-spotless.txt" + # Generate file-level annotations for GitHub Actions + if [[ -n "${BUGLINECOMMENTS}" ]]; then + # Extract files with violations: lines like "[ERROR] src/path/to/file.java" + # with leading whitespace after [ERROR] + ${GREP} '^\[ERROR\][[:space:]]\+[^[:space:]]' "${logfile}" \ + | ${SED} 's/^\[ERROR\][[:space:]]*//g' \ + | while read -r file; do + echo "${file}:1:Spotless formatting required, run mvn spotless:apply" + done > "${linecommentsfile}" + if [[ -s "${linecommentsfile}" ]]; then + bugsystem_linecomments_queue spotless "${linecommentsfile}" + fi + fi + + add_vote_table_v2 -1 spotless \ + "@@BASE@@/${repostatus}-spotless.txt" \ + "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix." return 1 fi - add_vote_table +1 spotless "${repostatus} has no errors when running spotless:check." + add_vote_table_v2 +1 spotless "" "${repostatus} has no errors when running spotless:check." return 0 } diff --git a/dev-support/integration-test/integration-test.Jenkinsfile b/dev-support/integration-test/integration-test.Jenkinsfile new file mode 100644 index 000000000000..d4490bc3311b --- /dev/null +++ b/dev-support/integration-test/integration-test.Jenkinsfile @@ -0,0 +1,404 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +pipeline { + agent { + node { + label 'hbase' + } + } + triggers { + pollSCM('@daily') + } + options { + buildDiscarder(logRotator(numToKeepStr: '20')) + timeout (time: 16, unit: 'HOURS') + timestamps() + skipDefaultCheckout() + disableConcurrentBuilds() + } + environment { + HADOOP_VERSIONS = "2.10.2,3.2.4,3.3.5,3.3.6,3.4.0,3.4.1,3.4.2,3.4.3" + BASEDIR = "${env.WORKSPACE}/component" + } + parameters { + booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') + } + stages { + stage('scm-checkout') { + steps { + dir('component') { + checkout scm + } + } + } + // This is meant to mimic what a release manager will do to create RCs. + // See http://hbase.apache.org/book.html#maven.release + // TODO (HBASE-23870): replace this with invocation of the release tool + stage ('packaging test') { + steps { + sh '''#!/bin/bash -e + echo "Setting up directories" + rm -rf "output-srctarball" && mkdir "output-srctarball" + rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball" + rm -rf ".m2-for-repo" && mkdir ".m2-for-repo" + rm -rf ".m2-for-src" && mkdir ".m2-for-src" + ''' + sh '''#!/bin/bash -e + rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine" + echo "got the following saved stats in 'output-srctarball/machine'" + ls -lh "output-srctarball/machine" + ''' + sh '''#!/bin/bash -e + echo "Checking the steps for an RM to make a source artifact, then a binary artifact." + docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" . + docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ + -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" --workdir=/hbase hbase-integration-test \ + "component/dev-support/integration-test/source-artifact.sh" \ + --intermediate-file-dir output-srctarball \ + --unpack-temp-dir unpacked_src_tarball \ + --maven-m2-initial .m2-for-repo \ + --maven-m2-src-build .m2-for-src \ + --clean-source-checkout \ + component + if [ $? -eq 0 ]; then + echo '(/) {color:green}+1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile + else + echo '(x) {color:red}-1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile + exit 1 + fi + ''' + echo "make sure we have proper hbase tarballs under hbase-assembly" + sh '''#!/bin/bash -e + if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v hadoop3 | wc -l) ]; then + echo '(x) {color:red}-1 testing binary artifact{color}\n-- source tarball did not produce the expected binaries.' >>output-srctarball/commentfile + exit 1 + fi + if [[ "${BRANCH_NAME}" == *"branch-2"* ]]; then + if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz | wc -l) ]; then + echo '(x) {color:red}-1 testing binary artifact{color}\n-- source tarball did not produce the expected hadoop3 binaries.' >>output-srctarball/commentfile + exit 1 + fi + fi + ''' + stash name: 'hbase-install', includes: "unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz" + } // steps + post { + always { + script { + def srcFile = "${env.WORKSPACE}/output-srctarball/hbase-src.tar.gz" + if (fileExists(srcFile)) { + echo "upload hbase-src.tar.gz to nightlies" + sshPublisher(publishers: [ + sshPublisherDesc(configName: 'Nightlies', + transfers: [ + sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", + sourceFiles: srcFile + ) + ] + ) + ]) + // remove the big src tarball, store the nightlies url in hbase-src.html + sh '''#!/bin/bash -e + SRC_TAR="${WORKSPACE}/output-srctarball/hbase-src.tar.gz" + echo "Remove ${SRC_TAR} for saving space" + rm -rf "${SRC_TAR}" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html" + ''' + } + } + archiveArtifacts artifacts: 'output-srctarball/*' + archiveArtifacts artifacts: 'output-srctarball/**/*' + } + } + } // packaging test + stage ('integration test matrix') { + matrix { + agent { + node { + label 'hbase' + } + } + axes { + axis { + name 'HADOOP_VERSION' + // matrix does not support dynamic axis values, so here we need to keep align with the + // above environment + values "2.10.2","3.2.4","3.3.5","3.3.6","3.4.0","3.4.1","3.4.2","3.4.3" + } + } + environment { + BASEDIR = "${env.WORKSPACE}/component" + OUTPUT_DIR = "output-integration-hadoop-${env.HADOOP_VERSION}" + } + when { + expression { + if (HADOOP_VERSION == '2.10.2') { + // only branch-2/branch-2.x need to run against hadoop2, here we also includes + // HBASE-XXXXX-branch-2 feature branch + return env.BRANCH_NAME.contains('branch-2') + } + if (HADOOP_VERSION == '3.2.4') { + // only branch-2.5 need to run against hadoop 3.2.4, here we also includes + // HBASE-XXXXX-branch-2.5 feature branch + return env.BRANCH_NAME.contains('branch-2.5') + } + return true + } + } + stages { + stage('scm-checkout') { + steps { + sh '''#!/bin/bash -e + echo "Setting up directories" + rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" + echo "(x) {color:red}-1 client integration test for ${HADOOP_VERSION}{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >${OUTPUT_DIR}/commentfile + rm -rf "unpacked_src_tarball" + rm -rf "hbase-install" && mkdir "hbase-install" + rm -rf "hbase-client" && mkdir "hbase-client" + rm -rf "hadoop-install" && mkdir "hadoop-install" + rm -rf "hbase-hadoop3-install" + rm -rf "hbase-hadoop3-client" + # remove old hadoop tarballs in workspace + rm -rf hadoop-*.tar.gz + ''' + dir('component') { + checkout scm + } + } // steps + } // scm-checkout + stage('install hadoop') { + steps { + dir("downloads-hadoop") { + sh '''#!/bin/bash -e + echo "Make sure we have a directory for downloading dependencies: $(pwd)" + ''' + sh '''#!/bin/bash -e + echo "Ensure we have a copy of Hadoop ${HADOOP_VERSION}" + "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --working-dir "${WORKSPACE}/downloads-hadoop" \ + --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ + --verify-tar-gz \ + "${WORKSPACE}/hadoop-${HADOOP_VERSION}-bin.tar.gz" \ + "hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz" + for stale in $(ls -1 "${WORKSPACE}"/hadoop-*.tar.gz | grep -v ${HADOOP_VERSION}); do + echo "Delete stale hadoop cache ${stale}" + rm -rf $stale + done + artifact=$(ls -1 "${WORKSPACE}"/hadoop-${HADOOP_VERSION}-bin.tar.gz | head -n 1) + tar --strip-components=1 -xzf "${artifact}" -C "${WORKSPACE}/hadoop-install" + if [[ ${HADOOP_VERSION} == 3.* ]]; then + # we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471 + ${BASEDIR}/dev-support/integration-test/patch-hadoop3.sh "${WORKSPACE}/hadoop-install" + fi + ''' + } // dir + } // steps + } // install hadoop + stage('install hbase') { + steps { + unstash 'hbase-install' + sh'''#!/bin/bash -e + install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v client-bin | grep -v hadoop3) + tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install" + client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-client-bin.tar.gz | grep -v hadoop3) + tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client" + if ls "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz &>/dev/null; then + echo "hadoop3 artifacts available, unpacking the hbase hadoop3 bin tarball into 'hbase-hadoop3-install' and the client hadoop3 tarball into 'hbase-hadoop3-client'" + mkdir hbase-hadoop3-install + mkdir hbase-hadoop3-client + hadoop3_install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz | grep -v client-bin) + tar --strip-component=1 -xzf "${hadoop3_install_artifact}" -C "hbase-hadoop3-install" + hadoop3_client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-client-bin.tar.gz) + tar --strip-component=1 -xzf "${hadoop3_client_artifact}" -C "hbase-hadoop3-client" + fi + ''' + } // steps + } + stage('integration test ') { + steps { + sh '''#!/bin/bash -e + hbase_install_dir="hbase-install" + hbase_client_dir="hbase-client" + if [[ ${HADOOP_VERSION} == 3.* ]] && [[ -d "hbase-hadoop3-install" ]]; then + echo "run hadoop3 client integration test against hbase hadoop3 binaries" + hbase_install_dir="hbase-hadoop3-install" + hbase_client_dir="hbase-hadoop3-client" + fi + java_home="/usr/lib/jvm/java-17" + hadoop_opts="--add-opens java.base/java.lang=ALL-UNNAMED" + if [[ ${HADOOP_VERSION} == 2.* ]]; then + java_home="/usr/lib/jvm/java-8" + hadoop_opts="" + fi + echo "Attempting to run an instance on top of Hadoop ${HADOOP_VERSION}." + # Create working dir + rm -rf "${OUTPUT_DIR}/non-shaded" && mkdir "${OUTPUT_DIR}/non-shaded" + docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" . + docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ + -u `id -u`:`id -g` -e JAVA_HOME="${java_home}" \ + -e HADOOP_OPTS="${hadoop_opts}" \ + --workdir=/hbase hbase-integration-test \ + component/dev-support/integration-test/pseudo-distributed-test.sh \ + --single-process \ + --working-dir ${OUTPUT_DIR}/non-shaded \ + --hbase-client-install ${hbase_client_dir} \ + ${hbase_install_dir} \ + hadoop-install/bin/hadoop \ + hadoop-install/share/hadoop/yarn/timelineservice \ + hadoop-install/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-install/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + hadoop-install/bin/mapred \ + >${OUTPUT_DIR}/hadoop.log 2>&1 + if [ $? -ne 0 ]; then + echo "(x) {color:red}-1 client integration test for ${HADOOP_VERSION}{color}\n--Failed when running client tests on top of Hadoop ${HADOOP_VERSION}. [see log for details|${BUILD_URL}/artifact/${OUTPUT_DIR}/hadoop.log]. (note that this means we didn't check the Hadoop ${HADOOP_VERSION} shaded client)" >${OUTPUT_DIR}/commentfile + exit 2 + fi + echo "(/) {color:green}+1 client integration test for ${HADOOP_VERSION} {color}" >${OUTPUT_DIR}/commentfile + if [[ ${HADOOP_VERSION} == 2.* ]] || [[ ${HADOOP_VERSION} == 3.2.* ]]; then + echo "skip running shaded hadoop client test for ${HADOOP_VERSION}" + exit 0 + fi + # Create working dir + rm -rf "${OUTPUT_DIR}/shaded" && mkdir "${OUTPUT_DIR}/shaded" + echo "Attempting to run an instance on top of Hadoop ${HADOOP_VERSION}, relying on the Hadoop client artifacts for the example client program." + docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ + -u `id -u`:`id -g` -e JAVA_HOME="${java_home}" \ + -e HADOOP_OPTS="${hadoop_opts}" \ + --workdir=/hbase hbase-integration-test \ + component/dev-support/integration-test/pseudo-distributed-test.sh \ + --single-process \ + --hadoop-client-classpath hadoop-install/share/hadoop/client/hadoop-client-api-*.jar:hadoop-install/share/hadoop/client/hadoop-client-runtime-*.jar \ + --working-dir ${OUTPUT_DIR}/shaded \ + --hbase-client-install ${hbase_client_dir} \ + ${hbase_install_dir} \ + hadoop-install/bin/hadoop \ + hadoop-install/share/hadoop/yarn/timelineservice \ + hadoop-install/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-install/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + hadoop-install/bin/mapred \ + >${OUTPUT_DIR}/hadoop-shaded.log 2>&1 + if [ $? -ne 0 ]; then + echo "(x) {color:red}-1 client integration testfor ${HADOOP_VERSION}{color}\n--Failed when running client tests on top of Hadoop ${HADOOP_VERSION} using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/${OUTPUT_DIR}/hadoop-shaded.log]." >> ${OUTPUT_DIR}/commentfile + exit 2 + fi + echo "(/) {color:green}+1 client integration test for ${HADOOP_VERSION} with shaded hadoop client{color}" >> ${OUTPUT_DIR}/commentfile + ''' + } // steps + post { + always { + stash name: "test-result-${env.HADOOP_VERSION}", includes: "${env.OUTPUT_DIR}/commentfile" + archiveArtifacts artifacts: "${env.OUTPUT_DIR}/*" + archiveArtifacts artifacts: "${env.OUTPUT_DIR}/**/*" + } // always + } // post + } // integration test + } // stages + } // matrix + } // integration test matrix + } // stages + post { + always { + script { + sh "printenv" + // wipe out all the output directories before unstashing + sh''' + echo "Clean up result directories" + rm -rf output-srctarball + rm -rf output-integration-hadoop-* + ''' + def results = [] + results.add('output-srctarball/commentfile') + for (hadoopVersion in getHadoopVersions(env.HADOOP_VERSIONS)) { + try { + unstash "test-result-${hadoopVersion}" + results.add("output-integration-hadoop-${hadoopVersion}/commentfile") + } catch (e) { + echo "unstash ${hadoopVersion} failed, ignore" + } + } + echo env.BRANCH_NAME + echo env.BUILD_URL + echo currentBuild.result + echo currentBuild.durationString + def comment = "Results for branch ${env.BRANCH_NAME}\n" + comment += "\t[build ${currentBuild.displayName} on builds.a.o|${env.BUILD_URL}]: " + if (currentBuild.result == null || currentBuild.result == "SUCCESS") { + comment += "(/) *{color:green}+1 overall{color}*\n" + } else { + comment += "(x) *{color:red}-1 overall{color}*\n" + // Ideally get the committer our of the change and @ mention them in the per-jira comment + } + comment += "----\ndetails (if available):\n\n" + echo "" + echo "[DEBUG] trying to aggregate step-wise results" + comment += results.collect { fileExists(file: it) ? readFile(file: it) : "" }.join("\n\n") + echo "[INFO] Comment:" + echo comment + echo "" + echo "[DEBUG] checking to see if feature branch" + def jiras = getJirasToComment(env.BRANCH_NAME, []) + if (jiras.isEmpty()) { + echo "[DEBUG] non-feature branch, checking change messages for jira keys." + echo "[INFO] There are ${currentBuild.changeSets.size()} change sets." + jiras = getJirasToCommentFromChangesets(currentBuild) + } + jiras.each { currentIssue -> + jiraComment issueKey: currentIssue, body: comment + } + } // script + } // always + } // post +} + +@NonCPS +List getHadoopVersions(String versions) { + return versions.split(',').collect { it.trim() }.findAll { it } as String[] +} + +import org.jenkinsci.plugins.workflow.support.steps.build.RunWrapper +@NonCPS +List getJirasToCommentFromChangesets(RunWrapper thisBuild) { + def seenJiras = [] + thisBuild.changeSets.each { cs -> + cs.getItems().each { change -> + CharSequence msg = change.msg + echo "change: ${change}" + echo " ${msg}" + echo " ${change.commitId}" + echo " ${change.author}" + echo "" + seenJiras = getJirasToComment(msg, seenJiras) + } + } + return seenJiras +} + +@NonCPS +List getJirasToComment(CharSequence source, List seen) { + source.eachMatch("HBASE-[0-9]+") { currentIssue -> + echo "[DEBUG] found jira key: ${currentIssue}" + if (currentIssue in seen) { + echo "[DEBUG] already commented on ${currentIssue}." + } else { + echo "[INFO] commenting on ${currentIssue}." + seen << currentIssue + } + } + return seen +} + diff --git a/dev-support/patch-hadoop3.sh b/dev-support/integration-test/patch-hadoop3.sh similarity index 83% rename from dev-support/patch-hadoop3.sh rename to dev-support/integration-test/patch-hadoop3.sh index fed5a535bcdd..b4c51ca9487d 100755 --- a/dev-support/patch-hadoop3.sh +++ b/dev-support/integration-test/patch-hadoop3.sh @@ -20,5 +20,5 @@ hadoop_dir=$1 sed -i "s/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"share\/hadoop\/tools\"}/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"\$HADOOP_TOOLS_HOME\/share\/hadoop\/tools\"}/g" "$hadoop_dir/libexec/hadoop-functions.sh" -sed -i "/hadoop_add_classpath \"\${junitjar}\"/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" "$hadoop_dir/bin/mapred" +sed -i "/HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.MiniHadoopClusterManager/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" "$hadoop_dir/bin/mapred" curl https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar -o "$hadoop_dir/share/hadoop/tools/lib/mockito-core-2.28.2.jar" diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/integration-test/pseudo-distributed-test.sh similarity index 96% rename from dev-support/hbase_nightly_pseudo-distributed-test.sh rename to dev-support/integration-test/pseudo-distributed-test.sh index 75f0ab9a9f8d..d9da9a38ef09 100755 --- a/dev-support/hbase_nightly_pseudo-distributed-test.sh +++ b/dev-support/integration-test/pseudo-distributed-test.sh @@ -308,7 +308,11 @@ fi if [ "${hadoop_version%.*.*}" -gt 2 ]; then echo "Verifying configs" - "${hadoop_exec}" --config "${working_dir}/hbase-conf/" conftest + hadoop_conf_files="" + for f in "${working_dir}"/hbase-conf/*-site.xml; do + hadoop_conf_files="$hadoop_conf_files -conffile $f" + done + "${hadoop_exec}" --config "${working_dir}/hbase-conf/" conftest $hadoop_conf_files fi if [ -n "${clean}" ]; then @@ -322,6 +326,14 @@ echo "Listing HDFS contents" redirect_and_run "${working_dir}/hadoop_cluster_smoke" \ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R / +if [ "${hadoop_version%.*.*}" -gt 2 ]; then + # for now, all hbase branches are compiled with hadoop 3.4.x when using hadoop-3.0 profile, where + # the protobuf library has been shaded and relocated, so we always need to use ProtobufRpcEngine2 + # at hbase side, even if the hadoop server side uses ProtobufRpcEngine, so here we will remove the + # config value to let the hadoop code pick the right rpc engine + sed -i "/.*ProtobufRpcEngine.*<\/property>/d" "${working_dir}/hbase-conf/core-site.xml" +fi + echo "Starting up HBase" HBASE_CONF_DIR="${working_dir}/hbase-conf/" HBASE_LOG_DIR="${working_dir}" "${component_install}/bin/start-hbase.sh" @@ -413,7 +425,7 @@ HADOOP_CLASSPATH="${hbase_dep_classpath}" redirect_and_run "${working_dir}/mr-im EOF echo "Verifying row count from import." -import_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1) +import_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | grep "row(s)" | awk '{print $1}') if [ ! "${import_rowcount}" -eq 48 ]; then echo "ERROR: Instead of finding 48 rows, we found ${import_rowcount}." exit 2 @@ -526,7 +538,7 @@ echo "Checking on results of example program." EOF echo "Verifying row count from example." -example_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1) +example_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | grep "row(s)" | awk '{print $1}') if [ "${example_rowcount}" -gt "1050" ]; then echo "Found ${example_rowcount} rows, which is enough to cover 48 for import, 1000 example's use of user table regions, 2 for example's use of meta/namespace regions, and 1 for example's count record" else diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/integration-test/source-artifact.sh similarity index 98% rename from dev-support/hbase_nightly_source-artifact.sh rename to dev-support/integration-test/source-artifact.sh index 372c39b8f887..1b54aaae0e3f 100755 --- a/dev-support/hbase_nightly_source-artifact.sh +++ b/dev-support/integration-test/source-artifact.sh @@ -34,6 +34,8 @@ function usage { exit 1 } +set -e + MVN="mvn" if ! command -v mvn &>/dev/null; then MVN=$MAVEN_HOME/bin/mvn @@ -218,8 +220,10 @@ function build_tarball { cd "${unpack_dir}" -${MVN} -Dmaven.repo.local="${m2_tarbuild}" help:active-profiles | grep -q hadoop-3.0 -if [ $? -ne 0 ]; then +if ${MVN} -Dmaven.repo.local="${m2_tarbuild}" help:active-profiles | grep -q hadoop-3.0; then + echo "The hadoop-3.0 profile is activated by default, build a default tarball." + build_tarball 0 +else echo "The hadoop-3.0 profile is not activated by default, build a default tarball first." # use java 8 to build with hadoop2 JAVA_HOME="/usr/lib/jvm/java-8" build_tarball 0 @@ -236,7 +240,4 @@ if [ $? -ne 0 ]; then fi # move tarballs back mv "${unpack_dir}"/hbase-*-bin.tar.gz "${unpack_dir}"/hbase-assembly/target/ -else - echo "The hadoop-3.0 profile is activated by default, build a default tarball." - build_tarball 0 fi diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh index 2d8421fd756d..05ac0f89a404 100755 --- a/dev-support/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins_precommit_github_yetus.sh @@ -31,7 +31,6 @@ declare -i missing_env=0 declare -a required_envs=( # these ENV variables define the required API with Jenkinsfile_GitHub "ARCHIVE_PATTERN_LIST" - "BUILD_URL_ARTIFACTS" "DOCKERFILE" "GITHUB_PASSWORD" "GITHUB_USER" @@ -39,7 +38,6 @@ declare -a required_envs=( "PLUGINS" "SET_JAVA_HOME" "SOURCEDIR" - "TESTS_FILTER" "YETUSDIR" "AUTHOR_IGNORE_LIST" "BLANKS_EOL_IGNORE_FILE" @@ -53,6 +51,12 @@ for required_env in "${required_envs[@]}"; do fi done +# BUILD_URL_ARTIFACTS is required for Jenkins but set in personality for GitHub Actions +if [[ "${GITHUB_ACTIONS}" != "true" ]] && [[ -z "${BUILD_URL_ARTIFACTS}" ]]; then + echo "[ERROR] Required environment variable 'BUILD_URL_ARTIFACTS' is not set." + missing_env=${missing_env}+1 +fi + if [ ${missing_env} -gt 0 ]; then echo "[ERROR] Please set the required environment variables before invoking. If this error is " \ "on Jenkins, then please file a JIRA about the error." @@ -91,7 +95,11 @@ YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt") YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html") # enable writing back to Github YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}") -YETUS_ARGS+=("--github-write-comment") +# GitHub Actions fork PRs cannot write comments (GITHUB_TOKEN has no PR write permission) +# Jenkins can write comments via its own credentials +if [[ "${GITHUB_ACTIONS}" != "true" ]]; then + YETUS_ARGS+=("--github-write-comment") +fi # auto-kill any surefire stragglers during unit test runs YETUS_ARGS+=("--reapermode=kill") # set relatively high limits for ASF machines @@ -103,7 +111,9 @@ YETUS_ARGS+=("--spotbugs-strict-precheck") # rsync these files back into the archive dir YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}") # URL for user-side presentation in reports and such to our artifacts -YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}") +if [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then + YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}") +fi # plugins to enable YETUS_ARGS+=("--plugins=${PLUGINS},-findbugs") # run in docker mode and specifically point to our @@ -115,15 +125,17 @@ YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}") YETUS_ARGS+=("--author-ignore-list=${AUTHOR_IGNORE_LIST}") YETUS_ARGS+=("--blanks-eol-ignore-file=${BLANKS_EOL_IGNORE_FILE}") YETUS_ARGS+=("--blanks-tabs-ignore-file=${BLANKS_TABS_IGNORE_FILE}*") -YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}") +if [[ -n "${TESTS_FILTER}" ]]; then + YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}") +fi YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh") YETUS_ARGS+=("--quick-hadoopcheck") if [[ "${SKIP_ERRORPRONE}" = "true" ]]; then # skip error prone YETUS_ARGS+=("--skip-errorprone") fi -# effectively treat dev-support as a custom maven module -YETUS_ARGS+=("--skip-dirs=dev-support") +# Exclude non-code directories from module detection to avoid triggering full builds +YETUS_ARGS+=("--skip-dirs=dev-support,.github,bin,conf") # For testing with specific hadoop version. Activates corresponding profile in maven runs. if [[ -n "${HADOOP_PROFILE}" ]]; then # Master has only Hadoop3 support. We don't need to activate any profile. @@ -156,6 +168,10 @@ fi if [[ -n "${JAVA8_HOME}" ]]; then YETUS_ARGS+=("--java8-home=${JAVA8_HOME}") fi +# Test profile for running specific test categories (e.g., runDevTests, runLargeTests-wave1) +if [[ -n "${TEST_PROFILE}" ]]; then + YETUS_ARGS+=("--test-profile=${TEST_PROFILE}") +fi echo "Launching yetus with command line:" echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}" diff --git a/dev-support/spotbugs-exclude.xml b/dev-support/spotbugs-exclude.xml index 2f0684eff4d7..17b8d2cbdedd 100644 --- a/dev-support/spotbugs-exclude.xml +++ b/dev-support/spotbugs-exclude.xml @@ -271,4 +271,16 @@ + + + + + + + + + diff --git a/dev-support/yetus_console_to_md.py b/dev-support/yetus_console_to_md.py new file mode 100644 index 000000000000..bee5512eec71 --- /dev/null +++ b/dev-support/yetus_console_to_md.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Convert Apache Yetus console output to Markdown format. +""" +import os +import re +import sys +from io import TextIOWrapper +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +# Vote to emoji mapping +VOTE_EMOJI = { + '+1': '✅', + '-1': '❌', + '0': '🆗', + '+0': '🆗', + '-0': '⚠️' +} + + +def convert_vote(vote: str) -> str: + """Convert vote string to emoji.""" + return VOTE_EMOJI.get(vote, vote) + + +def is_runtime(text: str) -> bool: + """Check if text is a runtime like '41m 24s'.""" + return bool(re.match(r'^\d+m\s+\d+s$', text)) + + +def parse_table_row(line: str) -> Tuple[str, str, str, str]: + """ + Parse a table row and return tuple of cell values. + Returns exactly 4 columns: (vote, subsystem, runtime, comment) + """ + parts = line.split('|') + # Remove first empty element (from leading |) + parts = parts[1:] if len(parts) > 1 else [] + + # Take first 4 columns and strip whitespace + result: List[str] = [p.strip() for p in parts[:4]] + + # Pad to 4 columns if needed + while len(result) < 4: + result.append('') + + return result[0], result[1], result[2], result[3] + + +def is_results_section_start(line: str) -> bool: + """Check if line indicates the start of Results section.""" + return bool(re.search(r'^\[\w+] Results:', line.strip())) + + +def is_tests_run_summary(line: str) -> bool: + """Check if line is the Tests run summary line.""" + return bool(re.search(r'^\[\w+] Tests run:', line.strip())) + + +def parse_results_section( + f: TextIOWrapper, + failures: List[str], + flakes: List[str], + errors: List[str] +) -> None: + """ + Parse the Results section within a patch-unit file. + """ + current_error_type = None + while line := f.readline(): + stripped = line.strip() + + # Section end markers + if is_tests_run_summary(line): + return + + # Detect error type sections + if re.search(r'^\[\w+] Failures:', stripped): + current_error_type = failures + elif re.search(r'^\[\w+] Flakes:', stripped): + current_error_type = flakes + elif re.search(r'^\[\w+] Errors:', stripped): + current_error_type = errors + else: + # Parse test entries + if current_error_type is not None: + test_match = re.search( + r'^\[\w+]\s+((?:org\.)?\S+\.(?:\w+\.)*\w+\.\w+)', + stripped + ) + if test_match: + test_name = test_match.group(1) + if 'test' in test_name.lower(): + current_error_type.append(test_name) + + +def skip_to_results_section(f: TextIOWrapper) -> bool: + """ + Skip the io stream to the Results section. + After calling this method, the TextIOWrapper will locate at the next line of "Results: " + + Returns: + True if we find a results section, False if we have reached the EOF + """ + while line := f.readline(): + if is_results_section_start(line): + return True + return False + + +def scan_all_tests(dir: Path) -> Dict[str, str]: + """ + Scan the archiver dir to find all the tests and their module + + Returns: + Dict mapping test name to module name + """ + module = None + module_to_test_name = {} + for dirpath, _, filenames in os.walk(dir): + if len(filenames) > 0: + # /archiver//target/surefire-reports + module = dirpath.split(os.sep)[-3] + for filename in filenames: + match = re.match(r'(org\.apache\.[^-]+)\.txt', filename) + if match: + module_to_test_name[match.group(1)] = module + return module_to_test_name + + +def parse_patch_unit_file( + file_path: Path, + failures: List[str], + flakes: List[str], + errors: List[str] +) -> None: + """ + Parse a patch-unit-*.txt file and extract failed tests by module. + """ + with open(file_path, 'r') as f: + while skip_to_results_section(f): + parse_results_section(f, failures, flakes, errors) + + +def get_module(test_name: str, test_name_to_module: Dict[str, str]) -> str: + rindex_of_bracket = test_name.rfind('[') + if rindex_of_bracket > 0: + # parameterized test, remove the tailing parameters + test_name = test_name[:rindex_of_bracket] + + module = test_name_to_module.get(test_name) + if module: + return module + + # usually the failed test name has the method name suffix, but the test_name_to_module only + # contains class name, so let's try to remove the last part and try again + rindex_of_dot = test_name.rfind('.') + if rindex_of_dot > 0: + test_name = test_name[:rindex_of_dot] + + module = test_name_to_module.get(test_name) + if module: + return module + return 'default' + + +def increase(module_to_count: Dict[str, int], module: str) -> None: + if module in module_to_count: + module_to_count[module] += 1 + else: + module_to_count[module] = 1 + + +def add_to_details(test_name: str, module: str, error_type: str, + details: Dict[str, Dict[str, List[str]]]) -> None: + if module not in details: + error_type_to_tests = {} + details[module] = error_type_to_tests + else: + error_type_to_tests = details[module] + + if error_type in error_type_to_tests: + error_type_to_tests[error_type].append(test_name) + else: + error_type_to_tests[error_type] = [test_name] + + +def process_failed_tests( + error_type: str, + failed_tests: List[str], + module_to_test_name: Dict[str, str], + counts: Dict[str, Dict[str, int]], + details: Dict[str, Dict[str, List[str]]] +) -> None: + for test_name in failed_tests: + module = get_module(test_name, module_to_test_name) + increase(counts[error_type], module) + add_to_details(test_name, module, error_type, details) + + +def aggregate_failed_tests(yetus_dir: Path) -> Tuple[ + Dict[str, Dict[str, int]], Dict[str, Dict[str, List[str]]]]: + """ + Aggregate failed tests from all patch-unit-*.txt files. + + Returns: + Tuple of: + - counts: {error_type: {module: count}} + - details: {module: {error_type: [test_names]}} + """ + patch_files = list(yetus_dir.glob('patch-unit-*.txt')) + + if not patch_files: + return {}, {} + + # Aggregate results from all files + failures = [] + flakes = [] + errors = [] + + for patch_file in patch_files: + parse_patch_unit_file(patch_file, failures, flakes, errors) + + if not failures and not flakes and not errors: + return {}, {} + + counts = {'Failures': {}, 'Flakes': {}, 'Errors': {}} + details = {} + module_to_test_name = scan_all_tests(yetus_dir / 'archiver') + process_failed_tests('Failures', failures, module_to_test_name, counts, details) + process_failed_tests('Flakes', flakes, module_to_test_name, counts, details) + process_failed_tests('Errors', errors, module_to_test_name, counts, details) + + return dict(counts), dict(details) + + +def generate_failed_tests_table( + counts: Dict[str, Dict[str, int]], + details: Dict[str, Dict[str, List[str]]] +) -> List[str]: + """Generate the Failed Tests HTML table.""" + total_failures = sum(sum(m.values()) for m in counts.values()) + if total_failures == 0: + return [] + + content = [ + '\n## Failed Tests\n\n', + '\n', + '\n', + '\n' + ] + + error_types = ['Failures', 'Flakes', 'Errors'] + + for error_type in error_types: + if error_type not in counts: + continue + + modules = counts[error_type] + total_count = sum(modules.values()) + num_modules = len(modules) + + first_row = True + for module in sorted(modules.keys()): + tests = details.get(module, {}).get(error_type, []) + tests_str = '
'.join(sorted(set(tests))) if tests else '' + + if first_row: + content.append( + f'' + f'' + f'\n' + ) + first_row = False + else: + content.append(f'\n') + + content.extend(['\n', '
Error TypeCountModuleTests
{error_type}{total_count}{module}{tests_str}
{module}{tests_str}
\n']) + + return content + + +def collect_continuation_lines( + lines: List[str], + start_idx: int +) -> Tuple[List[str], int]: + """ + Collect continuation lines for a table row. + + Args: + lines: All lines from the file + start_idx: Index to start checking from + + Returns: + Tuple of (list of comment parts, next index to process) + """ + comment_parts = [] + i = start_idx + + while i < len(lines): + line = lines[i] + stripped = line.strip() + + if not stripped.startswith('|'): + break + + if '|| Subsystem || Report/Notes ||' in line: + break + + vote, _, runtime, comment = parse_table_row(line) + + # Stop at new data row + if vote in VOTE_EMOJI: + break + + # Empty vote/subsystem means continuation or separator + if not vote: + if comment: + comment_parts.append(comment) + i += 1 + elif runtime and is_runtime(runtime): + break + else: + i += 1 + else: + break + + return comment_parts, i + + +def process_first_table(lines: List[str], start_idx: int) -> Tuple[List[str], int]: + """ + Process the first table (Vote, Subsystem, Runtime, Comment). + + Returns: + Tuple of (Markdown lines, next index to process) + """ + content = [ + '\n', + '| Vote | Subsystem | Runtime | Comment |\n', + '|------|-----------|---------|---------|\n' + ] + + i = start_idx + + # Skip the original separator line + if i < len(lines) and '===' in lines[i]: + i += 1 + + while i < len(lines): + line = lines[i] + stripped = line.strip() + + if '|| Subsystem || Report/Notes ||' in line: + break + + if stripped.startswith('+--'): + i += 1 + continue + + if not stripped.startswith('|'): + i += 1 + continue + + vote, subsystem, runtime, comment = parse_table_row(line) + + # Section header (vote and subsystem are empty) + if not vote and not subsystem: + if comment: + content.append(f'| | | | {comment} |\n') + elif runtime and is_runtime(runtime): + content.append(f'| | | {runtime} | |\n') + i += 1 + continue + + # Data row with vote + if vote in VOTE_EMOJI: + vote_emoji = convert_vote(vote) + comment_parts = [comment] if comment else [] + + continuation_parts, i = collect_continuation_lines(lines, i + 1) + comment_parts.extend(continuation_parts) + + comment_text = ' '.join(comment_parts) + content.append(f'| {vote_emoji} | {subsystem} | {runtime} | {comment_text} |\n') + continue + + # Other cases, skip + i += 1 + + return content, i + + +def process_second_table(lines: List[str], start_idx: int) -> Tuple[List[str], int]: + """ + Process the second table (Subsystem, Report/Notes). + + Returns: + Tuple of (Markdown lines, next index to process) + """ + content = [ + '\n## Subsystem Reports\n\n', + '| Subsystem | Report/Notes |\n', + '|-----------|------------|\n' + ] + + i = start_idx + + # Skip the original separator line + if i < len(lines) and '===' in lines[i]: + i += 1 + + while i < len(lines): + line = lines[i] + stripped = line.strip() + + if not stripped.startswith('|'): + break + + # Split by | and get non-empty parts + parts = [p.strip() for p in stripped.split('|') if p.strip()] + if len(parts) >= 2: + content.append(f'| {parts[0]} | {parts[1]} |\n') + + i += 1 + + return content, i + + +def convert_console_to_markdown(input_dir: str, output_file: Optional[str] = None) -> str: + """Convert Yetus console output to Markdown format.""" + input_path = Path(input_dir) + + if not input_path.is_dir(): + print(f'Error: Input path "{input_dir}" is not a directory', file=sys.stderr) + sys.exit(1) + + console_file = input_path / 'console.txt' + if not console_file.exists(): + print(f'Error: console.txt not found in "{input_dir}"', file=sys.stderr) + sys.exit(1) + + with open(console_file, 'r') as f: + lines = f.readlines() + + content = [] + i = 0 + + while i < len(lines): + line = lines[i] + stripped = line.strip() + + if stripped == '-1 overall': + content.append(f'

❌ {stripped}

\n') + i += 1 + elif stripped == '+1 overall': + content.append(f'

✅ {stripped}

\n') + i += 1 + elif '| Vote |' in line and 'Subsystem' in line: + table_content, i = process_first_table(lines, i + 1) + content.extend(table_content) + + counts, details = aggregate_failed_tests(input_path) + if counts: + content.extend(generate_failed_tests_table(counts, details)) + elif '|| Subsystem || Report/Notes ||' in line: + table_content, i = process_second_table(lines, i + 1) + content.extend(table_content) + else: + i += 1 + + result = ''.join(content) + + if output_file: + with open(output_file, 'w') as f: + f.write(result) + print(f'Converted {input_dir} to {output_file}', file=sys.stderr) + else: + print(result, end='') + + return result + + +def main(): + if len(sys.argv) < 2: + print(f'Usage: {sys.argv[0]} [output_file]', file=sys.stderr) + print( + f' input_directory: Directory containing console.txt and optional patch-unit-*.txt files', + file=sys.stderr) + print(f' If output_file is not provided, output goes to stdout', file=sys.stderr) + sys.exit(1) + + input_dir = sys.argv[1] + output_file = sys.argv[2] if len(sys.argv) > 2 else None + + if not Path(input_dir).exists(): + print(f'Error: Input directory "{input_dir}" does not exist', file=sys.stderr) + sys.exit(1) + + convert_console_to_markdown(input_dir, output_file) + + +if __name__ == '__main__': + main() diff --git a/hbase-annotations/pom.xml b/hbase-annotations/pom.xml index 2dc7c6102722..d977f3c78c6f 100644 --- a/hbase-annotations/pom.xml +++ b/hbase-annotations/pom.xml @@ -40,6 +40,13 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + true + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java index d9bae8490637..b0e259e1f9e2 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java @@ -36,4 +36,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface ClientTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.ClientTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/CoprocessorTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/CoprocessorTests.java index a168adec08af..2dc143e944a0 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/CoprocessorTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/CoprocessorTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface CoprocessorTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.CoprocessorTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FilterTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FilterTests.java index 84f346baaea2..1b45b583c182 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FilterTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FilterTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface FilterTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.FilterTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FlakeyTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FlakeyTests.java index c23bfa298b36..0cb861979e08 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FlakeyTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FlakeyTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface FlakeyTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.FlakeyTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IOTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IOTests.java index 8eee0e6ae4b9..be55b3829e52 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IOTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IOTests.java @@ -36,4 +36,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface IOTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.IOTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IntegrationTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IntegrationTests.java index 4e555b73fedb..0003cd1db511 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IntegrationTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IntegrationTests.java @@ -34,4 +34,5 @@ * @see LargeTests */ public interface IntegrationTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.IntegrationTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/LargeTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/LargeTests.java index b47e5bab9a46..3a24764e706a 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/LargeTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/LargeTests.java @@ -33,4 +33,5 @@ * @see IntegrationTests */ public interface LargeTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.LargeTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MapReduceTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MapReduceTests.java index 0e68ab3c0340..ac5b05e30704 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MapReduceTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MapReduceTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface MapReduceTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MapReduceTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MasterTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MasterTests.java index 5dcf51b27e59..0ad843493ec1 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MasterTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MasterTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface MasterTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MasterTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MediumTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MediumTests.java index d1f836ec0049..548f655c774e 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MediumTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MediumTests.java @@ -32,4 +32,5 @@ * @see IntegrationTests */ public interface MediumTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MediumTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MetricsTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MetricsTests.java index 27beaacf963e..c6985d6b95cc 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MetricsTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MetricsTests.java @@ -21,4 +21,5 @@ * Tag a test that covers our metrics handling. */ public interface MetricsTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MetricsTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MiscTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MiscTests.java index 695042e801bf..b7b7ad4c3f66 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MiscTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MiscTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface MiscTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MiscTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RPCTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RPCTests.java index 929bd6487edf..71a24d5d5dd6 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RPCTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RPCTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface RPCTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.RPCTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RegionServerTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RegionServerTests.java index 3439afa76eba..d79691d6fac6 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RegionServerTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RegionServerTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface RegionServerTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.RegionServerTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ReplicationTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ReplicationTests.java index df606c960c25..74c65a57982d 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ReplicationTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ReplicationTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface ReplicationTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.ReplicationTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RestTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RestTests.java index a648b4c39e03..9a73fde57e2c 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RestTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RestTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface RestTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.RestTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SecurityTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SecurityTests.java index a4e55ad3aba0..939c25c05ff4 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SecurityTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SecurityTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface SecurityTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.SecurityTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SmallTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SmallTests.java index 64d2bce381b6..54e16d7ad1ae 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SmallTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SmallTests.java @@ -30,4 +30,5 @@ * @see IntegrationTests */ public interface SmallTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.SmallTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowMapReduceTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowMapReduceTests.java index d1f433b9719d..dac933ec78e4 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowMapReduceTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowMapReduceTests.java @@ -36,4 +36,6 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface VerySlowMapReduceTests { + public static final String TAG = + "org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowRegionServerTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowRegionServerTests.java index f556979e5b6a..1583de103e38 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowRegionServerTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowRegionServerTests.java @@ -36,4 +36,6 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface VerySlowRegionServerTests { + public static final String TAG = + "org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ZKTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ZKTests.java index 9fa0579ed47e..a318b388ef72 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ZKTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ZKTests.java @@ -22,4 +22,5 @@ * {@code RecoverableZooKeeper}, not for tests which depend on ZooKeeper. */ public interface ZKTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.ZKTests"; } diff --git a/hbase-archetypes/hbase-client-project/pom.xml b/hbase-archetypes/hbase-client-project/pom.xml index 6d70cb345275..67ca8d55ee7c 100644 --- a/hbase-archetypes/hbase-client-project/pom.xml +++ b/hbase-archetypes/hbase-client-project/pom.xml @@ -87,8 +87,18 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-archetypes/hbase-client-project/src/test/java/org/apache/hbase/archetypes/exemplars/client/TestHelloHBase.java b/hbase-archetypes/hbase-client-project/src/test/java/org/apache/hbase/archetypes/exemplars/client/TestHelloHBase.java index b08ecf7ab1b5..defd9a2b0fd7 100644 --- a/hbase-archetypes/hbase-client-project/src/test/java/org/apache/hbase/archetypes/exemplars/client/TestHelloHBase.java +++ b/hbase-archetypes/hbase-client-project/src/test/java/org/apache/hbase/archetypes/exemplars/client/TestHelloHBase.java @@ -17,10 +17,10 @@ */ package org.apache.hbase.archetypes.exemplars.client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.client.Admin; @@ -30,30 +30,25 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Unit testing for HelloHBase. */ -@Category(MediumTests.class) +@Tag(MediumTests.TAG) public class TestHelloHBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHelloHBase.class); - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { TEST_UTIL.startMiniCluster(1); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } @@ -66,11 +61,11 @@ public void testNamespaceExists() throws Exception { Admin admin = TEST_UTIL.getAdmin(); exists = HelloHBase.namespaceExists(admin, NONEXISTENT_NAMESPACE); - assertEquals("#namespaceExists failed: found nonexistent namespace.", false, exists); + assertFalse(exists, "#namespaceExists failed: found nonexistent namespace."); admin.createNamespace(NamespaceDescriptor.create(EXISTING_NAMESPACE).build()); exists = HelloHBase.namespaceExists(admin, EXISTING_NAMESPACE); - assertEquals("#namespaceExists failed: did NOT find existing namespace.", true, exists); + assertTrue(exists, "#namespaceExists failed: did NOT find existing namespace."); admin.deleteNamespace(EXISTING_NAMESPACE); } @@ -80,10 +75,10 @@ public void testCreateNamespaceAndTable() throws Exception { HelloHBase.createNamespaceAndTable(admin); boolean namespaceExists = HelloHBase.namespaceExists(admin, HelloHBase.MY_NAMESPACE_NAME); - assertEquals("#createNamespaceAndTable failed to create namespace.", true, namespaceExists); + assertTrue(namespaceExists, "#createNamespaceAndTable failed to create namespace."); boolean tableExists = admin.tableExists(HelloHBase.MY_TABLE_NAME); - assertEquals("#createNamespaceAndTable failed to create table.", true, tableExists); + assertTrue(tableExists, "#createNamespaceAndTable failed to create table."); admin.disableTable(HelloHBase.MY_TABLE_NAME); admin.deleteTable(HelloHBase.MY_TABLE_NAME); @@ -98,7 +93,7 @@ public void testPutRowToTable() throws IOException { HelloHBase.putRowToTable(table); Result row = table.get(new Get(HelloHBase.MY_ROW_ID)); - assertEquals("#putRowToTable failed to store row.", false, row.isEmpty()); + assertFalse(row.isEmpty(), "#putRowToTable failed to store row."); TEST_UTIL.deleteTable(HelloHBase.MY_TABLE_NAME); admin.deleteNamespace(HelloHBase.MY_NAMESPACE_NAME); @@ -114,7 +109,7 @@ public void testDeleteRow() throws IOException { HelloHBase.MY_FIRST_COLUMN_QUALIFIER, Bytes.toBytes("xyz"))); HelloHBase.deleteRow(table); Result row = table.get(new Get(HelloHBase.MY_ROW_ID)); - assertEquals("#deleteRow failed to delete row.", true, row.isEmpty()); + assertTrue(row.isEmpty(), "#deleteRow failed to delete row."); TEST_UTIL.deleteTable(HelloHBase.MY_TABLE_NAME); admin.deleteNamespace(HelloHBase.MY_NAMESPACE_NAME); diff --git a/hbase-archetypes/hbase-shaded-client-project/pom.xml b/hbase-archetypes/hbase-shaded-client-project/pom.xml index 69832f8f6326..caec8fe032a7 100644 --- a/hbase-archetypes/hbase-shaded-client-project/pom.xml +++ b/hbase-archetypes/hbase-shaded-client-project/pom.xml @@ -87,8 +87,18 @@ runtime - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-archetypes/hbase-shaded-client-project/src/test/java/org/apache/hbase/archetypes/exemplars/shaded_client/TestHelloHBase.java b/hbase-archetypes/hbase-shaded-client-project/src/test/java/org/apache/hbase/archetypes/exemplars/shaded_client/TestHelloHBase.java index f87d9d7c700b..9a4be8b61cc4 100644 --- a/hbase-archetypes/hbase-shaded-client-project/src/test/java/org/apache/hbase/archetypes/exemplars/shaded_client/TestHelloHBase.java +++ b/hbase-archetypes/hbase-shaded-client-project/src/test/java/org/apache/hbase/archetypes/exemplars/shaded_client/TestHelloHBase.java @@ -17,10 +17,10 @@ */ package org.apache.hbase.archetypes.exemplars.shaded_client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.client.Admin; @@ -30,30 +30,25 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Unit testing for HelloHBase. */ -@Category(MediumTests.class) +@Tag(MediumTests.TAG) public class TestHelloHBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHelloHBase.class); - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { TEST_UTIL.startMiniCluster(1); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } @@ -66,11 +61,11 @@ public void testNamespaceExists() throws Exception { Admin admin = TEST_UTIL.getAdmin(); exists = HelloHBase.namespaceExists(admin, NONEXISTENT_NAMESPACE); - assertEquals("#namespaceExists failed: found nonexistent namespace.", false, exists); + assertFalse(exists, "#namespaceExists failed: found nonexistent namespace."); admin.createNamespace(NamespaceDescriptor.create(EXISTING_NAMESPACE).build()); exists = HelloHBase.namespaceExists(admin, EXISTING_NAMESPACE); - assertEquals("#namespaceExists failed: did NOT find existing namespace.", true, exists); + assertTrue(exists, "#namespaceExists failed: did NOT find existing namespace."); admin.deleteNamespace(EXISTING_NAMESPACE); } @@ -80,10 +75,10 @@ public void testCreateNamespaceAndTable() throws Exception { HelloHBase.createNamespaceAndTable(admin); boolean namespaceExists = HelloHBase.namespaceExists(admin, HelloHBase.MY_NAMESPACE_NAME); - assertEquals("#createNamespaceAndTable failed to create namespace.", true, namespaceExists); + assertTrue(namespaceExists, "#createNamespaceAndTable failed to create namespace."); boolean tableExists = admin.tableExists(HelloHBase.MY_TABLE_NAME); - assertEquals("#createNamespaceAndTable failed to create table.", true, tableExists); + assertTrue(tableExists, "#createNamespaceAndTable failed to create table."); admin.disableTable(HelloHBase.MY_TABLE_NAME); admin.deleteTable(HelloHBase.MY_TABLE_NAME); @@ -98,7 +93,7 @@ public void testPutRowToTable() throws IOException { HelloHBase.putRowToTable(table); Result row = table.get(new Get(HelloHBase.MY_ROW_ID)); - assertEquals("#putRowToTable failed to store row.", false, row.isEmpty()); + assertFalse(row.isEmpty(), "#putRowToTable failed to store row."); TEST_UTIL.deleteTable(HelloHBase.MY_TABLE_NAME); admin.deleteNamespace(HelloHBase.MY_NAMESPACE_NAME); @@ -114,7 +109,7 @@ public void testDeleteRow() throws IOException { HelloHBase.MY_FIRST_COLUMN_QUALIFIER, Bytes.toBytes("xyz"))); HelloHBase.deleteRow(table); Result row = table.get(new Get(HelloHBase.MY_ROW_ID)); - assertEquals("#deleteRow failed to delete row.", true, row.isEmpty()); + assertTrue(row.isEmpty(), "#deleteRow failed to delete row."); TEST_UTIL.deleteTable(HelloHBase.MY_TABLE_NAME); admin.deleteNamespace(HelloHBase.MY_NAMESPACE_NAME); diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index 459d3d2cb922..8ff8fcdf2a71 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -249,13 +249,13 @@ - ${build.year} - ${license.debug.print.included} - ${license.bundles.dependencies} - ${license.bundles.jquery} - ${license.bundles.vega} - ${license.bundles.logo} - ${license.bundles.bootstrap} + ${build.year} + ${license.debug.print.included} + ${license.bundles.dependencies} + ${license.bundles.jquery} + ${license.bundles.vega} + ${license.bundles.logo} + ${license.bundles.bootstrap} ${project.groupId}:hbase-resource-bundle:${project.version} @@ -388,8 +388,24 @@ hbase-rsgroup - junit - junit + org.junit.jupiter + junit-jupiter-api + compile + + + org.junit.jupiter + junit-jupiter-engine + compile + + + org.junit.jupiter + junit-jupiter-params + compile + + + org.junit.vintage + junit-vintage-engine + compile org.mockito diff --git a/hbase-asyncfs/pom.xml b/hbase-asyncfs/pom.xml index aa9787292eb6..ad2e7d805df5 100644 --- a/hbase-asyncfs/pom.xml +++ b/hbase-asyncfs/pom.xml @@ -69,8 +69,18 @@ true - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java index 61ab42d81be8..de200a36abff 100644 --- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java +++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java @@ -337,8 +337,8 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc if (evt instanceof IdleStateEvent) { IdleStateEvent e = (IdleStateEvent) evt; if (e.state() == READER_IDLE) { - failed(ctx.channel(), - () -> new IOException("Timeout(" + timeoutMs + "ms) waiting for response")); + failed(ctx.channel(), () -> new IOException("Timeout(" + timeoutMs + + "ms) waiting for response from datanode " + ctx.channel().remoteAddress())); } else if (e.state() == WRITER_IDLE) { PacketHeader heartbeat = new PacketHeader(4, 0, HEART_BEAT_SEQNO, false, 0, false); int len = heartbeat.getSerializedSize(); diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java index b93768ae0849..42bb40aeaf5a 100644 --- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java +++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java @@ -436,7 +436,8 @@ protected void channelRead0(ChannelHandlerContext ctx, BlockOpResponseProto resp if (PipelineAck.isRestartOOBStatus(pipelineStatus)) { throw new IOException("datanode " + dnInfo + " is restarting"); } - String logInfo = "ack with firstBadLink as " + resp.getFirstBadLink(); + String logInfo = "ack with firstBadLink as " + resp.getFirstBadLink() + " from datanode " + + dnInfo.getHostName() + "/" + dnInfo.getInfoAddr(); if (resp.getStatus() != Status.SUCCESS) { if (resp.getStatus() == Status.ERROR_ACCESS_TOKEN) { throw new InvalidBlockTokenException("Got access token error" + ", status message " @@ -469,8 +470,9 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { if (evt instanceof IdleStateEvent && ((IdleStateEvent) evt).state() == READER_IDLE) { - promise - .tryFailure(new IOException("Timeout(" + timeoutMs + "ms) waiting for response")); + promise.tryFailure( + new IOException("Timeout(" + timeoutMs + "ms) waiting for response from datanode " + + dnInfo.getHostName() + "/" + dnInfo.getInfoAddr())); } else { super.userEventTriggered(ctx, evt); } @@ -647,7 +649,8 @@ private static FanOutOneBlockAsyncDFSOutput createOutput(DistributedFileSystem d } catch (Exception e) { // exclude the broken DN next time toExcludeNodes.add(datanodeInfo); - excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, "connect error"); + excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, + ExcludeDatanodeManager.ExcludeCause.CONNECT_ERROR.getCause()); throw e; } } diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java index a39c4fba7919..3650854e6e18 100644 --- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java +++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java @@ -607,7 +607,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { if (evt instanceof IdleStateEvent && ((IdleStateEvent) evt).state() == READER_IDLE) { - promise.tryFailure(new IOException("Timeout(" + timeoutMs + "ms) waiting for response")); + promise.tryFailure(new IOException("Timeout(" + timeoutMs + + "ms) waiting for response from datanode " + ctx.channel().remoteAddress())); } else { super.userEventTriggered(ctx, evt); } diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/ExcludeDatanodeManager.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/ExcludeDatanodeManager.java index 61f75582a1c9..7bed67a94be5 100644 --- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/ExcludeDatanodeManager.java +++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/ExcludeDatanodeManager.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -52,7 +53,7 @@ public class ExcludeDatanodeManager implements ConfigurationObserver { "hbase.regionserver.async.wal.exclude.datanode.info.ttl.hour"; public static final int DEFAULT_WAL_EXCLUDE_DATANODE_TTL = 6; // 6 hours - private volatile Cache excludeDNsCache; + private volatile Cache> excludeDNsCache; private final int maxExcludeDNCount; private final Configuration conf; // This is a map of providerId->StreamSlowMonitor @@ -78,7 +79,7 @@ public ExcludeDatanodeManager(Configuration conf) { public boolean tryAddExcludeDN(DatanodeInfo datanodeInfo, String cause) { boolean alreadyMarkedSlow = getExcludeDNs().containsKey(datanodeInfo); if (!alreadyMarkedSlow) { - excludeDNsCache.put(datanodeInfo, EnvironmentEdgeManager.currentTime()); + excludeDNsCache.put(datanodeInfo, new Pair<>(cause, EnvironmentEdgeManager.currentTime())); LOG.info( "Added datanode: {} to exclude cache by [{}] success, current excludeDNsCache size={}", datanodeInfo, cause, excludeDNsCache.size()); @@ -95,7 +96,31 @@ public StreamSlowMonitor getStreamSlowMonitor(String name) { return streamSlowMonitors.computeIfAbsent(key, k -> new StreamSlowMonitor(conf, key, this)); } - public Map getExcludeDNs() { + /** + * Enumerates the reasons for excluding a datanode from certain operations. Each enum constant + * represents a specific cause leading to exclusion. + */ + public enum ExcludeCause { + CONNECT_ERROR("connect error"), + SLOW_PACKET_ACK("slow packet ack"); + + private final String cause; + + ExcludeCause(String cause) { + this.cause = cause; + } + + public String getCause() { + return cause; + } + + @Override + public String toString() { + return cause; + } + } + + public Map> getExcludeDNs() { return excludeDNsCache.asMap(); } diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java index c415706aa6af..a4b80fc64560 100644 --- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java +++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java @@ -156,7 +156,8 @@ public void checkProcessTimeAndSpeed(DatanodeInfo datanodeInfo, long packetDataL + "lastAckTimestamp={}, monitor name: {}", datanodeInfo, packetDataLen, processTimeMs, unfinished, lastAckTimestamp, this.name); if (addSlowAckData(datanodeInfo, packetDataLen, processTimeMs)) { - excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, "slow packet ack"); + excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, + ExcludeDatanodeManager.ExcludeCause.SLOW_PACKET_ACK.getCause()); } } } diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSTestBase.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSTestBase.java index 0bac2fb00a3d..4d69006a63a4 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSTestBase.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSTestBase.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,6 +88,11 @@ private static void createDirsAndSetProperties() throws IOException { createDirAndSetProperty("dfs.journalnode.edits.dir"); createDirAndSetProperty("dfs.provided.aliasmap.inmemory.leveldb.dir"); createDirAndSetProperty("fs.s3a.committer.staging.tmp.path"); + + // disable metrics logger since it depend on commons-logging internal classes and we do not want + // commons-logging on our classpath + UTIL.getConfiguration().setInt(DFSConfigKeys.DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY, 0); + UTIL.getConfiguration().setInt(DFSConfigKeys.DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_KEY, 0); } protected static void startMiniDFSCluster(int servers) throws IOException { diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestExcludeDatanodeManager.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestExcludeDatanodeManager.java index f7ca1639ec60..2a6f517c8378 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestExcludeDatanodeManager.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestExcludeDatanodeManager.java @@ -17,27 +17,23 @@ */ package org.apache.hadoop.hbase.io.asyncfs; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager; import org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor; +import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestExcludeDatanodeManager { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestExcludeDatanodeManager.class); - @Test public void testExcludeSlowDNBySpeed() { Configuration conf = HBaseConfiguration.create(); diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java index f0910684eddf..63be7bc9cf55 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java @@ -21,9 +21,9 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.FileNotFoundException; import java.io.IOException; @@ -38,7 +38,6 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager; import org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor; @@ -49,15 +48,12 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.ipc.RemoteException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.FixMethodOrder; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.junit.runners.MethodSorters; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,14 +63,10 @@ import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -@Category({ MiscTests.class, MediumTests.class }) +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) public class TestFanOutOneBlockAsyncDFSOutput extends AsyncFSTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFanOutOneBlockAsyncDFSOutput.class); - private static final Logger LOG = LoggerFactory.getLogger(TestFanOutOneBlockAsyncDFSOutput.class); private static DistributedFileSystem FS; private static EventLoopGroup EVENT_LOOP_GROUP; @@ -83,10 +75,9 @@ public class TestFanOutOneBlockAsyncDFSOutput extends AsyncFSTestBase { private static StreamSlowMonitor MONITOR; - @Rule - public TestName name = new TestName(); + private Path file; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.getConfiguration().setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, READ_TIMEOUT_MS); startMiniDFSCluster(3); @@ -96,7 +87,7 @@ public static void setUp() throws Exception { MONITOR = StreamSlowMonitor.create(UTIL.getConfiguration(), "testMonitor"); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { if (EVENT_LOOP_GROUP != null) { EVENT_LOOP_GROUP.shutdownGracefully().get(); @@ -136,13 +127,17 @@ static void writeAndVerify(FileSystem fs, Path f, AsyncFSOutput out) } } + @BeforeEach + public void setUpBeforeEach(TestInfo testInfo) { + file = new Path("/" + testInfo.getTestMethod().get().getName()); + } + @Test public void test() throws IOException, InterruptedException, ExecutionException { - Path f = new Path("/" + name.getMethodName()); EventLoop eventLoop = EVENT_LOOP_GROUP.next(); - FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, - false, (short) 3, FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true); - writeAndVerify(FS, f, out); + FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, file, + true, false, (short) 3, FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true); + writeAndVerify(FS, file, out); } /** @@ -151,10 +146,9 @@ public void test() throws IOException, InterruptedException, ExecutionException */ @Test public void test0Recover() throws IOException, InterruptedException, ExecutionException { - Path f = new Path("/" + name.getMethodName()); EventLoop eventLoop = EVENT_LOOP_GROUP.next(); - FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, - false, (short) 3, FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true); + FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, file, + true, false, (short) 3, FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true); byte[] b = new byte[10]; Bytes.random(b); out.write(b, 0, b.length); @@ -162,17 +156,12 @@ public void test0Recover() throws IOException, InterruptedException, ExecutionEx // restart one datanode which causes one connection broken CLUSTER.restartDataNode(0); out.write(b, 0, b.length); - try { - out.flush(false).get(); - fail("flush should fail"); - } catch (ExecutionException e) { - // we restarted one datanode so the flush should fail - LOG.info("expected exception caught", e); - } + ExecutionException e = assertThrows(ExecutionException.class, () -> out.flush(false).get()); + LOG.info("expected exception caught", e); out.recoverAndClose(null); - assertEquals(b.length, FS.getFileStatus(f).getLen()); + assertEquals(b.length, FS.getFileStatus(file).getLen()); byte[] actual = new byte[b.length]; - try (FSDataInputStream in = FS.open(f)) { + try (FSDataInputStream in = FS.open(file)) { in.readFully(actual); } assertArrayEquals(b, actual); @@ -180,13 +169,12 @@ public void test0Recover() throws IOException, InterruptedException, ExecutionEx @Test public void testHeartbeat() throws IOException, InterruptedException, ExecutionException { - Path f = new Path("/" + name.getMethodName()); EventLoop eventLoop = EVENT_LOOP_GROUP.next(); - FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, - false, (short) 3, FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true); + FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, file, + true, false, (short) 3, FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true); Thread.sleep(READ_TIMEOUT_MS * 2); // the connection to datanode should still alive. - writeAndVerify(FS, f, out); + writeAndVerify(FS, file, out); } /** @@ -194,16 +182,13 @@ public void testHeartbeat() throws IOException, InterruptedException, ExecutionE */ @Test public void testCreateParentFailed() throws IOException { - Path f = new Path("/" + name.getMethodName() + "/test"); + Path f = new Path(file, "test"); EventLoop eventLoop = EVENT_LOOP_GROUP.next(); - try { - FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, false, (short) 3, - FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true); - fail("should fail with parent does not exist"); - } catch (RemoteException e) { - LOG.info("expected exception caught", e); - assertThat(e.unwrapRemoteException(), instanceOf(FileNotFoundException.class)); - } + RemoteException e = assertThrows(RemoteException.class, + () -> FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, false, (short) 3, + FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true)); + LOG.info("expected exception caught", e); + assertThat(e.unwrapRemoteException(), instanceOf(FileNotFoundException.class)); } @Test @@ -264,19 +249,18 @@ public void testExcludeFailedConnectToDatanode() @Test public void testWriteLargeChunk() throws IOException, InterruptedException, ExecutionException { - Path f = new Path("/" + name.getMethodName()); EventLoop eventLoop = EVENT_LOOP_GROUP.next(); - FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, - false, (short) 3, 1024 * 1024 * 1024, eventLoop, CHANNEL_CLASS, MONITOR, true); + FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, file, + true, false, (short) 3, 1024 * 1024 * 1024, eventLoop, CHANNEL_CLASS, MONITOR, true); byte[] b = new byte[50 * 1024 * 1024]; Bytes.random(b); out.write(b); consume(out.flush(false)); assertEquals(b.length, out.flush(false).get().longValue()); out.close(); - assertEquals(b.length, FS.getFileStatus(f).getLen()); + assertEquals(b.length, FS.getFileStatus(file).getLen()); byte[] actual = new byte[b.length]; - try (FSDataInputStream in = FS.open(f)) { + try (FSDataInputStream in = FS.open(file)) { in.readFully(actual); } assertArrayEquals(b, actual); diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutputHang.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutputHang.java index 7f6535a93a93..4f70aaab421c 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutputHang.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutputHang.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.io.asyncfs; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.util.ArrayList; import java.util.Iterator; @@ -28,7 +28,6 @@ import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; @@ -37,13 +36,10 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,13 +61,10 @@ * in this test class we use the default value for timeout which is 60 seconds and it is enough for * this test. */ -@Category({ MiscTests.class, MediumTests.class }) +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) public class TestFanOutOneBlockAsyncDFSOutputHang extends AsyncFSTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFanOutOneBlockAsyncDFSOutputHang.class); - private static final Logger LOG = LoggerFactory.getLogger(TestFanOutOneBlockAsyncDFSOutputHang.class); @@ -85,10 +78,7 @@ public class TestFanOutOneBlockAsyncDFSOutputHang extends AsyncFSTestBase { private static FanOutOneBlockAsyncDFSOutput OUT; - @Rule - public TestName name = new TestName(); - - @BeforeClass + @BeforeAll public static void setUp() throws Exception { startMiniDFSCluster(2); FS = CLUSTER.getFileSystem(); @@ -101,7 +91,7 @@ public static void setUp() throws Exception { FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, MONITOR, true); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { if (OUT != null) { OUT.recoverAndClose(null); diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLeaseRenewal.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLeaseRenewal.java index 71f84b63133c..5f032c82b69a 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLeaseRenewal.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLeaseRenewal.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.io.asyncfs; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -27,7 +27,6 @@ import java.lang.reflect.Method; import java.util.Optional; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; @@ -35,11 +34,10 @@ import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.DummyDFSOutputStream; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.MockedConstruction; import org.apache.hbase.thirdparty.io.netty.channel.Channel; @@ -54,19 +52,16 @@ *

* See HBASE-28955 for more details. */ -@Category({ MiscTests.class, MediumTests.class }) +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) public class TestLeaseRenewal extends AsyncFSTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLeaseRenewal.class); - private static DistributedFileSystem FS; private static EventLoopGroup EVENT_LOOP_GROUP; private static Class CHANNEL_CLASS; private static StreamSlowMonitor MONITOR; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { startMiniDFSCluster(3); FS = CLUSTER.getFileSystem(); @@ -75,7 +70,7 @@ public static void setUp() throws Exception { MONITOR = StreamSlowMonitor.create(UTIL.getConfiguration(), "testMonitor"); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { if (EVENT_LOOP_GROUP != null) { EVENT_LOOP_GROUP.shutdownGracefully().get(); diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java index ec87316e5f1b..b6535cdcfd11 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java @@ -21,29 +21,24 @@ import java.util.concurrent.ExecutionException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.CommonFSUtils; -import org.junit.AfterClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.io.netty.channel.Channel; import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestLocalAsyncOutput { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLocalAsyncOutput.class); - private static EventLoopGroup GROUP = new NioEventLoopGroup(); private static Class CHANNEL_CLASS = NioSocketChannel.class; @@ -52,7 +47,7 @@ public class TestLocalAsyncOutput { private static StreamSlowMonitor MONITOR; - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.cleanupTestDir(); GROUP.shutdownGracefully().get(); diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestOverwriteFileUnderConstruction.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestOverwriteFileUnderConstruction.java index 7a3a6de10f09..a5bb0de9b339 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestOverwriteFileUnderConstruction.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestOverwriteFileUnderConstruction.java @@ -19,8 +19,8 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; import java.io.FileNotFoundException; import java.io.IOException; @@ -28,49 +28,40 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; import org.apache.hadoop.ipc.RemoteException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; /** * Used to confirm that it is OK to overwrite a file which is being written currently. */ -@Category({ MiscTests.class, MediumTests.class }) +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) public class TestOverwriteFileUnderConstruction extends AsyncFSTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestOverwriteFileUnderConstruction.class); - private static FileSystem FS; - @Rule - public final TestName name = new TestName(); - - @BeforeClass + @BeforeAll public static void setUp() throws Exception { startMiniDFSCluster(3); FS = CLUSTER.getFileSystem(); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { shutdownMiniDFSCluster(); } @Test - public void testNotOverwrite() throws IOException { - Path file = new Path("/" + name.getMethodName()); + public void testNotOverwrite(TestInfo testInfo) throws IOException { + Path file = new Path("/" + testInfo.getDisplayName()); try (FSDataOutputStream out1 = FS.create(file)) { try { FS.create(file, false); @@ -83,8 +74,8 @@ public void testNotOverwrite() throws IOException { } @Test - public void testOverwrite() throws IOException { - Path file = new Path("/" + name.getMethodName()); + public void testOverwrite(TestInfo testInfo) throws IOException { + Path file = new Path("/" + testInfo.getDisplayName()); FSDataOutputStream out1 = FS.create(file); FSDataOutputStream out2 = FS.create(file, true); out1.write(2); diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java index e178d0960eae..a0e56546f439 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Properties; import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -39,7 +40,7 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderFactory; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.SecurityConstants; @@ -48,19 +49,14 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.params.provider.Arguments; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,17 +66,14 @@ import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; -@RunWith(Parameterized.class) -@Category({ MiscTests.class, LargeTests.class }) +@Tag(MiscTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "[{index}] protection = {0}, encryption = {1}, cipher = {2}") public class TestSaslFanOutOneBlockAsyncDFSOutput extends AsyncFSTestBase { private static final Logger LOG = LoggerFactory.getLogger(TestSaslFanOutOneBlockAsyncDFSOutput.class); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSaslFanOutOneBlockAsyncDFSOutput.class); - private static DistributedFileSystem FS; private static EventLoopGroup EVENT_LOOP_GROUP; @@ -105,29 +98,27 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput extends AsyncFSTestBase { private static StreamSlowMonitor MONITOR; - @Rule - public TestName name = new TestName(); - - @Parameter(0) - public String protection; + private String protection; + private String encryptionAlgorithm; + private String cipherSuite; - @Parameter(1) - public String encryptionAlgorithm; - - @Parameter(2) - public String cipherSuite; + public TestSaslFanOutOneBlockAsyncDFSOutput(String protection, String encryptionAlgorithm, + String cipherSuite) { + this.protection = protection; + this.encryptionAlgorithm = encryptionAlgorithm; + this.cipherSuite = cipherSuite; + } - @Parameters(name = "{index}: protection={0}, encryption={1}, cipherSuite={2}") - public static Iterable data() { - List params = new ArrayList<>(); + public static Stream parameters() { + List params = new ArrayList<>(); for (String protection : Arrays.asList("authentication", "integrity", "privacy")) { for (String encryptionAlgorithm : Arrays.asList("", "3des", "rc4")) { for (String cipherSuite : Arrays.asList("", CipherSuite.AES_CTR_NOPADDING.getName())) { - params.add(new Object[] { protection, encryptionAlgorithm, cipherSuite }); + params.add(Arguments.of(protection, encryptionAlgorithm, cipherSuite)); } } } - return params; + return params.stream(); } private static void setUpKeyProvider(Configuration conf) throws Exception { @@ -175,7 +166,7 @@ private static MiniKdc setupMiniKdc(File keytabFile) throws Exception { return kdc; } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { EVENT_LOOP_GROUP = new NioEventLoopGroup(); CHANNEL_CLASS = NioSocketChannel.class; @@ -193,7 +184,7 @@ public static void setUpBeforeClass() throws Exception { MONITOR = StreamSlowMonitor.create(UTIL.getConfiguration(), "testMonitor"); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { if (EVENT_LOOP_GROUP != null) { EVENT_LOOP_GROUP.shutdownGracefully().get(); @@ -214,8 +205,8 @@ private void createEncryptionZone() throws Exception { method.invoke(FS, entryptionTestDirOnTestFs, TEST_KEY_NAME); } - @Before - public void setUp() throws Exception { + @BeforeEach + public void setUp(TestInfo testInfo) throws Exception { UTIL.getConfiguration().set("dfs.data.transfer.protection", protection); if (StringUtils.isBlank(encryptionAlgorithm) && StringUtils.isBlank(cipherSuite)) { UTIL.getConfiguration().setBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, false); @@ -235,14 +226,14 @@ public void setUp() throws Exception { startMiniDFSCluster(3); FS = CLUSTER.getFileSystem(); - testDirOnTestFs = new Path("/" + name.getMethodName().replaceAll("[^0-9a-zA-Z]", "_")); + testDirOnTestFs = new Path("/" + testInfo.getDisplayName().replaceAll("[^0-9a-zA-Z]", "_")); FS.mkdirs(testDirOnTestFs); entryptionTestDirOnTestFs = new Path("/" + testDirOnTestFs.getName() + "_enc"); FS.mkdirs(entryptionTestDirOnTestFs); createEncryptionZone(); } - @After + @AfterEach public void tearDown() throws IOException { shutdownMiniDFSCluster(); } @@ -262,8 +253,8 @@ private void test(Path file) throws IOException, InterruptedException, Execution TestFanOutOneBlockAsyncDFSOutput.writeAndVerify(FS, file, out); } - @Test - public void test() throws IOException, InterruptedException, ExecutionException { + @TestTemplate + public void test() throws Exception { test(getTestFile()); test(getEncryptionTestFile()); } diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSendBufSizePredictor.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSendBufSizePredictor.java index 07fc3afbf2ff..1ddf6d64f96a 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSendBufSizePredictor.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSendBufSizePredictor.java @@ -17,22 +17,17 @@ */ package org.apache.hadoop.hbase.io.asyncfs; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestSendBufSizePredictor { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSendBufSizePredictor.class); - @Test public void test() { SendBufSizePredictor p = new SendBufSizePredictor(); diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/util/TestRecoverLeaseFSUtils.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/util/TestRecoverLeaseFSUtils.java index 354095d15ff5..d3a2ccdb401e 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/util/TestRecoverLeaseFSUtils.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/util/TestRecoverLeaseFSUtils.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hbase.util; import static org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.LEASE_RECOVERABLE_CLASS_NAME; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -30,34 +30,33 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test our recoverLease loop against mocked up filesystem. */ -@Category({ MiscTests.class, MediumTests.class }) +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) public class TestRecoverLeaseFSUtils { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRecoverLeaseFSUtils.class); - private static final HBaseCommonTestingUtility HTU = new HBaseCommonTestingUtility(); - static { + + private static Path FILE; + + @BeforeAll + public static void setUp() { Configuration conf = HTU.getConfiguration(); conf.setInt("hbase.lease.recovery.first.pause", 10); conf.setInt("hbase.lease.recovery.pause", 10); + FILE = new Path(HTU.getDataTestDir(), "file.txt"); } - private static Path FILE = new Path(HTU.getDataTestDir(), "file.txt"); - /** * Test recover lease eventually succeeding. */ @@ -80,11 +79,8 @@ public void testRecoverLease() throws IOException { } private interface FakeLeaseRecoverable { - @SuppressWarnings("unused") - boolean recoverLease(Path p) throws IOException; - @SuppressWarnings("unused") - boolean isFileClosed(Path p) throws IOException; + boolean recoverLease(Path p) throws IOException; } private static abstract class RecoverableFileSystem extends FileSystem @@ -93,11 +89,6 @@ private static abstract class RecoverableFileSystem extends FileSystem public boolean recoverLease(Path p) throws IOException { return true; } - - @Override - public boolean isFileClosed(Path p) throws IOException { - return true; - } } /** diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml index 7bbf59b1e96a..769e3c669c3f 100644 --- a/hbase-backup/pom.xml +++ b/hbase-backup/pom.xml @@ -163,8 +163,18 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java index 25055fd5e8e6..fc8e79011cdb 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java @@ -77,18 +77,10 @@ public interface BackupAdmin extends Closeable { void mergeBackups(String[] backupIds) throws IOException; /** - * Show backup history command - * @param n last n backup sessions - * @return list of backup info objects - * @throws IOException exception - */ - List getHistory(int n) throws IOException; - - /** - * Show backup history command with filters - * @param n last n backup sessions - * @param f list of filters - * @return list of backup info objects + * Retrieve info about the most recent backups. + * @param n number of backup infos desired + * @param f optional filters, only entries passing the filters will be returned + * @return a list of at most n entries, ordered from newest (most recent) to oldest (least recent) * @throws IOException exception */ List getHistory(int n, BackupInfo.Filter... f) throws IOException; diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java index d55a280b4aa4..c135e7e0dff9 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java @@ -22,6 +22,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_HFILE_LOCATION_RESOLVER; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_HFILE_LOCATION_RESOLVER_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; @@ -35,6 +37,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WAL_LOCATION_RESOLVER; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WAL_LOCATION_RESOLVER_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; @@ -159,6 +163,8 @@ protected void addOptions() { addOptWithArg(OPTION_PATH, OPTION_PATH_DESC); addOptWithArg(OPTION_KEEP, OPTION_KEEP_DESC); addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_DESC); + addOptWithArg(OPTION_WAL_LOCATION_RESOLVER, OPTION_WAL_LOCATION_RESOLVER_DESC); + addOptWithArg(OPTION_HFILE_LOCATION_RESOLVER, OPTION_HFILE_LOCATION_RESOLVER_DESC); } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java index c9a76bef2891..289e09ff9df8 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java @@ -23,7 +23,6 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.TableName; @@ -52,45 +51,57 @@ public class BackupHFileCleaner extends BaseHFileCleanerDelegate implements Abor private boolean stopped = false; private boolean aborted = false; private Connection connection; - // timestamp of most recent read from backup system table - private long prevReadFromBackupTbl = 0; - // timestamp of 2nd most recent read from backup system table - private long secondPrevReadFromBackupTbl = 0; + private volatile long previousCleaningCompletionTimestamp = 0; + private volatile Set bulkLoadedHFilesNeedingBackup = null; @Override - public Iterable getDeletableFiles(Iterable files) { + public void preClean() { if (stopped) { - return Collections.emptyList(); + return; } - - // We use filenames because the HFile will have been moved to the archive since it - // was registered. - final Set hfileFilenames = new HashSet<>(); try (BackupSystemTable tbl = new BackupSystemTable(connection)) { Set tablesIncludedInBackups = fetchFullyBackedUpTables(tbl); + Set hfileFilenames = new HashSet<>(); for (BulkLoad bulkLoad : tbl.readBulkloadRows(tablesIncludedInBackups)) { - hfileFilenames.add(new Path(bulkLoad.getHfilePath()).getName()); + String path = bulkLoad.getHfilePath(); + hfileFilenames.add(path.substring(path.lastIndexOf('/') + 1)); } - LOG.debug("Found {} unique HFile filenames registered as bulk loads.", hfileFilenames.size()); + bulkLoadedHFilesNeedingBackup = hfileFilenames; + LOG.debug("Cached {} unique HFile filenames registered as bulk loads.", + hfileFilenames.size()); } catch (IOException ioe) { - LOG.error( - "Failed to read registered bulk load references from backup system table, marking all files as non-deletable.", - ioe); + LOG.error("Failed to read registered bulk load references from backup system table, " + + "marking all files as non-deletable.", ioe); + bulkLoadedHFilesNeedingBackup = null; + } + } + + @Override + public void postClean() { + previousCleaningCompletionTimestamp = EnvironmentEdgeManager.currentTime(); + bulkLoadedHFilesNeedingBackup = null; + } + + @Override + public Iterable getDeletableFiles(Iterable files) { + if (stopped) { + return Collections.emptyList(); + } + + final Set hfilesNeedingBackup = bulkLoadedHFilesNeedingBackup; + if (hfilesNeedingBackup == null) { return Collections.emptyList(); } - secondPrevReadFromBackupTbl = prevReadFromBackupTbl; - prevReadFromBackupTbl = EnvironmentEdgeManager.currentTime(); + final long recentFileThreshold = previousCleaningCompletionTimestamp; return Iterables.filter(files, file -> { - // If the file is recent, be conservative and wait for one more scan of the bulk loads - if (file.getModificationTime() > secondPrevReadFromBackupTbl) { + if (file.getModificationTime() > recentFileThreshold) { LOG.debug("Preventing deletion due to timestamp: {}", file.getPath().toString()); return false; } - // A file can be deleted if it is not registered as a backup bulk load. String hfile = file.getPath().getName(); - if (hfileFilenames.contains(hfile)) { + if (hfilesNeedingBackup.contains(hfile)) { LOG.debug("Preventing deletion due to bulk load registration in backup system table: {}", file.getPath().toString()); return false; diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java index f0dc10b83619..28b65dca4941 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.function.Predicate; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.util.BackupUtils; @@ -46,13 +47,22 @@ public class BackupInfo implements Comparable { private static final Logger LOG = LoggerFactory.getLogger(BackupInfo.class); private static final int MAX_FAILED_MESSAGE_LENGTH = 1024; - public interface Filter { - /** - * Filter interface - * @param info backup info - * @return true if info passes filter, false otherwise - */ - boolean apply(BackupInfo info); + public interface Filter extends Predicate { + /** Returns true if the BackupInfo passes the filter, false otherwise */ + @Override + boolean test(BackupInfo backupInfo); + } + + public static Filter withRoot(String backupRoot) { + return info -> info.getBackupRootDir().equals(backupRoot); + } + + public static Filter withType(BackupType type) { + return info -> info.getType() == type; + } + + public static Filter withState(BackupState state) { + return info -> info.getState() == state; } /** @@ -61,8 +71,7 @@ public interface Filter { public enum BackupState { RUNNING, COMPLETE, - FAILED, - ANY + FAILED } /** @@ -139,8 +148,11 @@ public enum BackupPhase { private List incrBackupFileList; /** - * New region server log timestamps for table set after distributed log roll key - table name, - * value - map of RegionServer hostname -> last log rolled timestamp + * New region server log timestamps for table set after distributed log roll. The keys consist of + * all tables that are part of the backup chain of the backup root (not just the tables that were + * specified when creating the backup, which could be a subset). The value is a map of + * RegionServer hostname to the last log-roll timestamp, i.e. the point up to which logs are + * included in the backup. */ private Map> tableSetTimestampMap; @@ -423,6 +435,18 @@ public BackupProtos.BackupInfo toProtosBackupInfo() { builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name())); builder.setWorkersNumber(workers); builder.setBandwidth(bandwidth); + builder.setTotalBytesCopied(totalBytesCopied); + builder.setNoChecksumVerify(noChecksumVerify); + if (incrBackupFileList != null) { + builder.addAllIncrBackupFileList(incrBackupFileList); + } + if (incrTimestampMap != null) { + for (Entry> entry : incrTimestampMap.entrySet()) { + builder.putIncrTimestampMap(entry.getKey().getNameAsString(), + BackupProtos.BackupInfo.RSTimestampMap.newBuilder().putAllRsTimestamp(entry.getValue()) + .build()); + } + } return builder.build(); } @@ -495,7 +519,7 @@ public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { BackupInfo context = new BackupInfo(); context.setBackupId(proto.getBackupId()); context.setBackupTableInfoMap(toMap(proto.getBackupTableInfoList())); - context.setTableSetTimestampMap(getTableSetTimestampMap(proto.getTableSetTimestampMap())); + context.setTableSetTimestampMap(toTableTimestampMap(proto.getTableSetTimestampMap())); context.setCompleteTs(proto.getCompleteTs()); if (proto.hasFailedMessage()) { context.setFailedMsg(proto.getFailedMessage()); @@ -504,8 +528,13 @@ public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { context.setState(BackupInfo.BackupState.valueOf(proto.getBackupState().name())); } - context - .setHLogTargetDir(BackupUtils.getLogBackupDir(proto.getBackupRootDir(), proto.getBackupId())); + // Only incremental backups have a WAL target directory. Setting this unconditionally would + // hand FULL backups a non-null path that never existed, which cleanupHLogDir() would then + // try to delete. + if (BackupType.valueOf(proto.getBackupType().name()) == BackupType.INCREMENTAL) { + context.setHLogTargetDir( + BackupUtils.getLogBackupDir(proto.getBackupRootDir(), proto.getBackupId())); + } if (proto.hasBackupPhase()) { context.setPhase(BackupPhase.valueOf(proto.getBackupPhase().name())); @@ -518,6 +547,14 @@ public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { context.setType(BackupType.valueOf(proto.getBackupType().name())); context.setWorkers(proto.getWorkersNumber()); context.setBandwidth(proto.getBandwidth()); + context.setTotalBytesCopied(proto.getTotalBytesCopied()); + context.setNoChecksumVerify(proto.getNoChecksumVerify()); + if (proto.getIncrBackupFileListCount() > 0) { + context.setIncrBackupFileList(new ArrayList<>(proto.getIncrBackupFileListList())); + } + if (proto.getIncrTimestampMapCount() > 0) { + context.setIncrTimestampMap(toTableTimestampMap(proto.getIncrTimestampMapMap())); + } return context; } @@ -530,14 +567,13 @@ private static Map toMap(List> - getTableSetTimestampMap(Map map) { - Map> tableSetTimestampMap = new HashMap<>(); + toTableTimestampMap(Map map) { + Map> result = new HashMap<>(); for (Entry entry : map.entrySet()) { - tableSetTimestampMap.put(TableName.valueOf(entry.getKey()), - entry.getValue().getRsTimestampMap()); + result.put(TableName.valueOf(entry.getKey()), entry.getValue().getRsTimestampMap()); } - return tableSetTimestampMap; + return result; } public String getShortDescription() { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java index 57454d402173..881e7ccf9353 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java @@ -99,6 +99,14 @@ public interface BackupRestoreConstants { String OPTION_YARN_QUEUE_NAME_DESC = "Yarn queue name to run backup create command on"; String OPTION_YARN_QUEUE_NAME_RESTORE_DESC = "Yarn queue name to run backup restore command on"; + String OPTION_WAL_LOCATION_RESOLVER = "wal-location-resolver"; + String OPTION_WAL_LOCATION_RESOLVER_DESC = + "WAL file location resolver class for rack-aware incremental backup"; + + String OPTION_HFILE_LOCATION_RESOLVER = "hfile-location-resolver"; + String OPTION_HFILE_LOCATION_RESOLVER_DESC = + "HFile location resolver class for rack-aware bulk loading during incremental backup"; + String JOB_NAME_CONF_KEY = "mapreduce.job.name"; String BACKUP_CONFIG_STRING = BackupRestoreConstants.BACKUP_ENABLE_KEY + "=true\n" diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java index 6e52d312ab7b..d6ee80a40141 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java @@ -62,7 +62,7 @@ public Builder withToTables(TableName[] toTables) { return this; } - public Builder withOvewrite(boolean overwrite) { + public Builder withOverwrite(boolean overwrite) { request.setOverwrite(overwrite); return this; } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java index c36b398e5e86..bf7b9b199d5e 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java @@ -17,6 +17,11 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withRoot; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; +import static org.apache.hadoop.hbase.backup.BackupInfo.withType; +import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -74,7 +79,8 @@ public BackupInfo getBackupInfo(String backupId) throws IOException { BackupInfo backupInfo; try (final BackupSystemTable table = new BackupSystemTable(conn)) { if (backupId == null) { - ArrayList recentSessions = table.getBackupInfos(BackupState.RUNNING); + List recentSessions = + table.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (recentSessions.isEmpty()) { LOG.warn("No ongoing sessions found."); return null; @@ -111,7 +117,8 @@ public int deleteBackups(String[] backupIds) throws IOException { } // Step 2: Make sure there is no failed session - List list = sysTable.getBackupInfos(BackupState.RUNNING); + List list = + sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (list.size() != 0) { // ailed sessions found LOG.warn("Failed backup session found. Run backup repair tool first."); @@ -299,9 +306,16 @@ private void removeTableFromBackupImage(BackupInfo info, TableName tn, BackupSys private List getAffectedBackupSessions(BackupInfo backupInfo, TableName tn, BackupSystemTable table) throws IOException { LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn); + // A FAILED backup was never part of the backup chain — its state was rolled back + // via snapshot restore. No subsequent backup depends on it, so there are no + // affected sessions. + if (backupInfo.getState() == BackupState.FAILED) { + LOG.debug("Backup {} is in FAILED state, skipping cascade", backupInfo.getBackupId()); + return new ArrayList<>(); + } long ts = backupInfo.getStartTs(); List list = new ArrayList<>(); - List history = table.getBackupHistory(backupInfo.getBackupRootDir()); + List history = table.getBackupHistory(withRoot(backupInfo.getBackupRootDir())); // Scan from most recent to backupInfo // break when backupInfo reached for (BackupInfo info : history) { @@ -367,49 +381,10 @@ private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long return false; } - @Override - public List getHistory(int n) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List history = table.getBackupHistory(); - - if (history.size() <= n) { - return history; - } - - List list = new ArrayList<>(); - for (int i = 0; i < n; i++) { - list.add(history.get(i)); - } - return list; - } - } - @Override public List getHistory(int n, BackupInfo.Filter... filters) throws IOException { - if (filters.length == 0) { - return getHistory(n); - } - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List history = table.getBackupHistory(); - List result = new ArrayList<>(); - for (BackupInfo bi : history) { - if (result.size() == n) { - break; - } - - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(bi)) { - passed = false; - break; - } - } - if (passed) { - result.add(bi); - } - } - return result; + return table.getBackupHistory(NEW_TO_OLD, n, filters); } } @@ -672,7 +647,7 @@ private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table) // Filter 1 : backupRoot // Filter 2 : time range filter // Filter 3 : table filter - BackupInfo.Filter destinationFilter = info -> info.getBackupRootDir().equals(backupDest); + BackupInfo.Filter destinationFilter = withRoot(backupDest); BackupInfo.Filter timeRangeFilter = info -> { long time = info.getStartTs(); @@ -684,10 +659,10 @@ private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table) return !Collections.disjoint(allTables, tables); }; - BackupInfo.Filter typeFilter = info -> info.getType() == BackupType.INCREMENTAL; - BackupInfo.Filter stateFilter = info -> info.getState() == BackupState.COMPLETE; + BackupInfo.Filter typeFilter = withType(BackupType.INCREMENTAL); + BackupInfo.Filter stateFilter = withState(BackupState.COMPLETE); - List allInfos = table.getBackupHistory(-1, destinationFilter, timeRangeFilter, + List allInfos = table.getBackupHistory(destinationFilter, timeRangeFilter, tableFilter, typeFilter, stateFilter); if (allInfos.size() != allBackups.size()) { // Yes we have at least one hole in backup image sequence diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 66694f4384f4..0e3054bb505c 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -17,11 +17,13 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_HFILE_LOCATION_RESOLVER; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; @@ -37,10 +39,12 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WAL_LOCATION_RESOLVER; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; +import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD; import java.io.IOException; import java.net.URI; @@ -83,6 +87,14 @@ public final class BackupCommands { public final static String TOP_LEVEL_NOT_ALLOWED = "Top level (root) folder is not allowed to be a backup destination"; + // Configuration keys for location resolvers + // Must match WALPlayer.CONF_WAL_FILE_LOCATION_RESOLVER_CLASS + private static final String CONF_WAL_FILE_LOCATION_RESOLVER_CLASS = + "wal.backup.file.location.resolver.class"; + // Must match HFileInputFormat.CONF_HFILE_LOCATION_RESOLVER_CLASS + private static final String CONF_HFILE_LOCATION_RESOLVER_CLASS = + "hfile.backup.input.file.location.resolver.class"; + public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n" + "where COMMAND is one of:\n" + " create create a new backup image\n" + " delete delete an existing backup image\n" @@ -148,12 +160,23 @@ public void execute() throws IOException { getConf().set("mapreduce.job.queuename", queueName); } + if (cmdline.hasOption(OPTION_WAL_LOCATION_RESOLVER)) { + String resolverClass = cmdline.getOptionValue(OPTION_WAL_LOCATION_RESOLVER); + getConf().set(CONF_WAL_FILE_LOCATION_RESOLVER_CLASS, resolverClass); + } + + if (cmdline.hasOption(OPTION_HFILE_LOCATION_RESOLVER)) { + String resolverClass = cmdline.getOptionValue(OPTION_HFILE_LOCATION_RESOLVER); + getConf().set(CONF_HFILE_LOCATION_RESOLVER_CLASS, resolverClass); + } + // Create connection conn = ConnectionFactory.createConnection(getConf()); if (requiresNoActiveSession()) { // Check active session try (BackupSystemTable table = new BackupSystemTable(conn)) { - List sessions = table.getBackupInfos(BackupState.RUNNING); + List sessions = + table.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (sessions.size() > 0) { System.err.println("Found backup session in a RUNNING state: "); @@ -528,7 +551,8 @@ public void execute() throws IOException { if (backupId != null) { info = sysTable.readBackupInfo(backupId); } else { - List infos = sysTable.getBackupInfos(BackupState.RUNNING); + List infos = + sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (infos != null && infos.size() > 0) { info = infos.get(0); backupId = info.getBackupId(); @@ -594,18 +618,15 @@ private void executeDeleteOlderThan(CommandLine cmdline) throws IOException { throw new IOException(value + " is not an integer number"); } final long fdays = days; - BackupInfo.Filter dateFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - long currentTime = EnvironmentEdgeManager.currentTime(); - long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000; - return info.getCompleteTs() <= maxTsToDelete; - } + BackupInfo.Filter dateFilter = info -> { + long currentTime = EnvironmentEdgeManager.currentTime(); + long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000; + return info.getCompleteTs() <= maxTsToDelete; }; List history = null; try (final BackupSystemTable sysTable = new BackupSystemTable(conn); BackupAdminImpl admin = new BackupAdminImpl(conn)) { - history = sysTable.getBackupHistory(-1, dateFilter); + history = sysTable.getBackupHistory(dateFilter); String[] backupIds = convertToBackupIds(history); int deleted = admin.deleteBackups(backupIds); System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: " @@ -679,7 +700,8 @@ public void execute() throws IOException { final BackupSystemTable sysTable = new BackupSystemTable(conn)) { // Failed backup BackupInfo backupInfo; - List list = sysTable.getBackupInfos(BackupState.RUNNING); + List list = + sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (list.size() == 0) { // No failed sessions found System.out.println("REPAIR status: no failed sessions found." @@ -860,27 +882,21 @@ public void execute() throws IOException { int n = parseHistoryLength(); final TableName tableName = getTableName(); final String setName = getTableSetName(); - BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (tableName == null) { - return true; - } - - List names = info.getTableNames(); - return names.contains(tableName); + BackupInfo.Filter tableNameFilter = info -> { + if (tableName == null) { + return true; } - }; - BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (setName == null) { - return true; - } - String backupId = info.getBackupId(); - return backupId.startsWith(setName); + List names = info.getTableNames(); + return names.contains(tableName); + }; + BackupInfo.Filter tableSetFilter = info -> { + if (setName == null) { + return true; } + + String backupId = info.getBackupId(); + return backupId.startsWith(setName); }; Path backupRootPath = getBackupRootPath(); List history; @@ -888,7 +904,8 @@ public boolean apply(BackupInfo info) { // Load from backup system table super.execute(); try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { - history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter); + history = sysTable.getBackupHistory(tableNameFilter, tableSetFilter); + history = history.subList(0, Math.min(n, history.size())); } } else { // load from backup FS diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java index 5e5098a7b66a..dff3cc27b006 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; +import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD; + import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; @@ -248,7 +251,8 @@ public BackupInfo createBackupInfo(String backupId, BackupType type, List sessions = systemTable.getBackupInfos(BackupState.RUNNING); + List sessions = + systemTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (sessions.size() == 0) { return null; } @@ -332,26 +336,6 @@ public void finishBackupSession() throws IOException { systemTable.finishBackupExclusiveOperation(); } - /** - * Read the last backup start code (timestamp) of last successful backup. Will return null if - * there is no startcode stored in backup system table or the value is of length 0. These two - * cases indicate there is no successful backup completed so far. - * @return the timestamp of a last successful backup - * @throws IOException exception - */ - public String readBackupStartCode() throws IOException { - return systemTable.readBackupStartCode(backupInfo.getBackupRootDir()); - } - - /** - * Write the start code (timestamp) to backup system table. If passed in null, then write 0 byte. - * @param startCode start code - * @throws IOException exception - */ - public void writeBackupStartCode(Long startCode) throws IOException { - systemTable.writeBackupStartCode(startCode, backupInfo.getBackupRootDir()); - } - /** * Get the RS log information after the last log roll from backup system table. * @return RS log info @@ -370,16 +354,10 @@ public void deleteBulkLoadedRows(List rows) throws IOException { } /** - * Get all completed backup information (in desc order by time) - * @return history info of BackupCompleteData - * @throws IOException exception + * Get all backup information, ordered by descending start time. I.e. from newest to oldest. */ - public List getBackupHistory() throws IOException { - return systemTable.getBackupHistory(); - } - - public ArrayList getBackupHistory(boolean completed) throws IOException { - return systemTable.getBackupHistory(completed); + public List getBackupHistory(BackupInfo.Filter... filters) throws IOException { + return systemTable.getBackupHistory(filters); } /** diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index c2253a46d04b..6c3b351a6302 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withRoot; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; +import static org.apache.hadoop.hbase.backup.BackupInfo.withType; + import edu.umd.cs.findbugs.annotations.Nullable; import java.io.Closeable; import java.io.IOException; @@ -36,7 +40,9 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -54,7 +60,6 @@ import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.backup.BackupRestoreConstants; import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -76,6 +81,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.base.Splitter; import org.apache.hbase.thirdparty.com.google.common.collect.Iterators; @@ -402,12 +408,10 @@ private List processBulkLoadRowScan(Scan scan) throws IOException { TableName table = null; String fam = null; String path = null; - String region = null; - byte[] row = null; + byte[] row = CellUtil.cloneRow(res.current()); + String rowStr = Bytes.toString(row); + String region = BackupSystemTable.getRegionNameFromOrigBulkLoadRow(rowStr); for (Cell cell : res.listCells()) { - row = CellUtil.cloneRow(cell); - String rowStr = Bytes.toString(row); - region = BackupSystemTable.getRegionNameFromOrigBulkLoadRow(rowStr); if ( CellUtil.compareQualifiers(cell, BackupSystemTable.TBL_COL, 0, BackupSystemTable.TBL_COL.length) == 0 @@ -452,48 +456,6 @@ public BackupInfo readBackupInfo(String backupId) throws IOException { } } - /** - * Read the last backup start code (timestamp) of last successful backup. Will return null if - * there is no start code stored on hbase or the value is of length 0. These two cases indicate - * there is no successful backup completed so far. - * @param backupRoot directory path to backup destination - * @return the timestamp of last successful backup - * @throws IOException exception - */ - public String readBackupStartCode(String backupRoot) throws IOException { - LOG.trace("read backup start code from backup system table"); - - try (Table table = connection.getTable(tableName)) { - Get get = createGetForStartCode(backupRoot); - Result res = table.get(get); - if (res.isEmpty()) { - return null; - } - Cell cell = res.listCells().get(0); - byte[] val = CellUtil.cloneValue(cell); - if (val.length == 0) { - return null; - } - return new String(val, StandardCharsets.UTF_8); - } - } - - /** - * Write the start code (timestamp) to backup system table. If passed in null, then write 0 byte. - * @param startCode start code - * @param backupRoot root directory path to backup - * @throws IOException exception - */ - public void writeBackupStartCode(Long startCode, String backupRoot) throws IOException { - if (LOG.isTraceEnabled()) { - LOG.trace("write backup start code to backup system table " + startCode); - } - try (Table table = connection.getTable(tableName)) { - Put put = createPutForStartCode(startCode.toString(), backupRoot); - table.put(put); - } - } - /** * Exclusive operations are: create, delete, merge * @throws IOException if a table operation fails or an active backup exclusive operation is @@ -592,123 +554,14 @@ public void writeRegionServerLastLogRollResult(String server, Long ts, String ba } /** - * Get all completed backup information (in desc order by time) - * @param onlyCompleted true, if only successfully completed sessions - * @return history info of BackupCompleteData - * @throws IOException exception - */ - public ArrayList getBackupHistory(boolean onlyCompleted) throws IOException { - LOG.trace("get backup history from backup system table"); - - BackupState state = onlyCompleted ? BackupState.COMPLETE : BackupState.ANY; - ArrayList list = getBackupInfos(state); - return BackupUtils.sortHistoryListDesc(list); - } - - /** - * Get all backups history - * @return list of backup info - * @throws IOException if getting the backup history fails - */ - public List getBackupHistory() throws IOException { - return getBackupHistory(false); - } - - /** - * Get first n backup history records - * @param n number of records, if n== -1 - max number is ignored - * @return list of records - * @throws IOException if getting the backup history fails - */ - public List getHistory(int n) throws IOException { - List history = getBackupHistory(); - if (n == -1 || history.size() <= n) { - return history; - } - return Collections.unmodifiableList(history.subList(0, n)); - } - - /** - * Get backup history records filtered by list of filters. - * @param n max number of records, if n == -1 , then max number is ignored - * @param filters list of filters - * @return backup records - * @throws IOException if getting the backup history fails - */ - public List getBackupHistory(int n, BackupInfo.Filter... filters) throws IOException { - if (filters.length == 0) { - return getHistory(n); - } - - List history = getBackupHistory(); - List result = new ArrayList<>(); - for (BackupInfo bi : history) { - if (n >= 0 && result.size() == n) { - break; - } - - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(bi)) { - passed = false; - break; - } - } - if (passed) { - result.add(bi); - } - } - return result; - } - - /** - * Retrieve all table names that are part of any known backup + * Retrieve all table names that are part of any known completed backup */ public Set getTablesIncludedInBackups() throws IOException { - Set names = new HashSet<>(); - List infos = getBackupHistory(true); - for (BackupInfo info : infos) { - // Incremental backups have the same tables as the preceding full backups - if (info.getType() == BackupType.FULL) { - names.addAll(info.getTableNames()); - } - } - return names; - } - - /** - * Get history for backup destination - * @param backupRoot backup destination path - * @return List of backup info - * @throws IOException if getting the backup history fails - */ - public List getBackupHistory(String backupRoot) throws IOException { - ArrayList history = getBackupHistory(false); - for (Iterator iterator = history.iterator(); iterator.hasNext();) { - BackupInfo info = iterator.next(); - if (!backupRoot.equals(info.getBackupRootDir())) { - iterator.remove(); - } - } - return history; - } - - /** - * Get history for a table - * @param name table name - * @return history for a table - * @throws IOException if getting the backup history fails - */ - public List getBackupHistoryForTable(TableName name) throws IOException { - List history = getBackupHistory(); - List tableHistory = new ArrayList<>(); - for (BackupInfo info : history) { - List tables = info.getTableNames(); - if (tables.contains(name)) { - tableHistory.add(info); - } - } - return tableHistory; + // Incremental backups have the same tables as the preceding full backups + List infos = + getBackupHistory(withState(BackupState.COMPLETE), withType(BackupType.FULL)); + return infos.stream().flatMap(info -> info.getTableNames().stream()) + .collect(Collectors.toSet()); } /** @@ -722,7 +575,7 @@ public List getBackupHistoryForTable(TableName name) throws IOExcept */ public Map> getBackupHistoryForTableSet(Set set, String backupRoot) throws IOException { - List history = getBackupHistory(backupRoot); + List history = getBackupHistory(withRoot(backupRoot)); Map> tableHistoryMap = new HashMap<>(); for (BackupInfo info : history) { List tables = info.getTableNames(); @@ -738,16 +591,32 @@ public Map> getBackupHistoryForTableSet(Set getBackupHistory(BackupInfo.Filter... toInclude) throws IOException { + return getBackupHistory(Order.NEW_TO_OLD, Integer.MAX_VALUE, toInclude); + } + + /** + * Retrieves the first n entries of the sorted, filtered list of backup infos. + * @param order desired ordering of the results. + * @param n number of entries to return */ - public ArrayList getBackupInfos(BackupState state) throws IOException { + public List getBackupHistory(Order order, int n, BackupInfo.Filter... toInclude) + throws IOException { + Preconditions.checkArgument(n >= 0, "n should be >= 0"); LOG.trace("get backup infos from backup system table"); - Scan scan = createScanForBackupHistory(); - ArrayList list = new ArrayList<>(); + if (n == 0) { + return Collections.emptyList(); + } + + Predicate combinedPredicate = Stream.of(toInclude) + .map(filter -> (Predicate) filter).reduce(Predicate::and).orElse(x -> true); + + Scan scan = createScanForBackupHistory(order); + List list = new ArrayList<>(); try (Table table = connection.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) { @@ -755,10 +624,12 @@ public ArrayList getBackupInfos(BackupState state) throws IOExceptio while ((res = scanner.next()) != null) { res.advance(); BackupInfo context = cellToBackupInfo(res.current()); - if (state != BackupState.ANY && context.getState() != state) { - continue; + if (combinedPredicate.test(context)) { + list.add(context); + if (list.size() == n) { + break; + } } - list.add(context); } return list; } @@ -930,21 +801,17 @@ public void deleteIncrementalBackupTableSet(String backupRoot) throws IOExceptio /** * Checks if we have at least one backup session in backup system table This API is used by * BackupLogCleaner - * @return true, if - at least one session exists in backup system table table + * @return true, if at least one session exists in backup system table * @throws IOException exception */ public boolean hasBackupSessions() throws IOException { LOG.trace("Has backup sessions from backup system table"); - boolean result = false; - Scan scan = createScanForBackupHistory(); + Scan scan = createScanForBackupHistory(Order.OLD_TO_NEW); scan.setCaching(1); try (Table table = connection.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) { - if (scanner.next() != null) { - result = true; - } - return result; + return scanner.next() != null; } } @@ -1206,29 +1073,6 @@ private BackupInfo resultToBackupInfo(Result res) throws IOException { return cellToBackupInfo(cell); } - /** - * Creates Get operation to retrieve start code from backup system table - * @return get operation - * @throws IOException exception - */ - private Get createGetForStartCode(String rootPath) throws IOException { - Get get = new Get(rowkey(START_CODE_ROW, rootPath)); - get.addFamily(BackupSystemTable.META_FAMILY); - get.readVersions(1); - return get; - } - - /** - * Creates Put operation to store start code to backup system table - * @return put operation - */ - private Put createPutForStartCode(String startCode, String rootPath) { - Put put = new Put(rowkey(START_CODE_ROW, rootPath)); - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("startcode"), - Bytes.toBytes(startCode)); - return put; - } - /** * Creates Get to retrieve incremental backup table set from backup system table * @return get operation @@ -1247,7 +1091,10 @@ private Get createGetForIncrBackupTableSet(String backupRoot) throws IOException * @return put operation */ private Put createPutForIncrBackupTableSet(Set tables, String backupRoot) { - Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot)); + // added 1ms to prevent LostUpdate problem in case when deleteIncrementalBackupTableSet() + // executed very fast + long ts = EnvironmentEdgeManager.currentTime() + 1; + Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot), ts); for (TableName table : tables) { put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(table.getNameAsString()), EMPTY_VALUE); @@ -1268,15 +1115,23 @@ private Delete createDeleteForIncrBackupTableSet(String backupRoot) { /** * Creates Scan operation to load backup history + * @param order order of the scan results * @return scan operation */ - private Scan createScanForBackupHistory() { + private Scan createScanForBackupHistory(Order order) { Scan scan = new Scan(); byte[] startRow = Bytes.toBytes(BACKUP_INFO_PREFIX); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.withStartRow(startRow); - scan.withStopRow(stopRow); + if (order == Order.NEW_TO_OLD) { + byte[] stopRow = Arrays.copyOf(startRow, startRow.length); + stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); + scan.setReversed(true); + scan.withStartRow(stopRow, false); + scan.withStopRow(startRow); + } else if (order == Order.OLD_TO_NEW) { + scan.setStartStopRowForPrefixScan(startRow); + } else { + throw new IllegalArgumentException("Unsupported order: " + order); + } scan.addFamily(BackupSystemTable.SESSIONS_FAMILY); scan.readVersions(1); return scan; @@ -1403,9 +1258,7 @@ public static void restoreFromSnapshot(Connection conn) throws IOException { try (Admin admin = conn.getAdmin()) { String snapshotName = BackupSystemTable.getSnapshotName(conf); if (snapshotExists(admin, snapshotName)) { - admin.disableTable(BackupSystemTable.getTableName(conf)); - admin.restoreSnapshot(snapshotName); - admin.enableTable(BackupSystemTable.getTableName(conf)); + admin.restoreBackupSystemTable(snapshotName); LOG.debug("Done restoring backup system table"); } else { // Snapshot does not exists, i.e completeBackup failed after @@ -1604,12 +1457,12 @@ static Scan createScanForOrigBulkLoadedFiles(@Nullable TableName table) { static String getTableNameFromOrigBulkLoadRow(String rowStr) { // format is bulk : namespace : table : region : file - return Iterators.get(Splitter.onPattern(BLK_LD_DELIM).split(rowStr).iterator(), 1); + return Iterators.get(Splitter.on(BLK_LD_DELIM).split(rowStr).iterator(), 1); } static String getRegionNameFromOrigBulkLoadRow(String rowStr) { // format is bulk : namespace : table : region : file - List parts = Splitter.onPattern(BLK_LD_DELIM).splitToList(rowStr); + List parts = Splitter.on(BLK_LD_DELIM).splitToList(rowStr); Iterator i = parts.iterator(); int idx = 3; if (parts.size() == 4) { @@ -1617,7 +1470,7 @@ static String getRegionNameFromOrigBulkLoadRow(String rowStr) { idx = 2; } String region = Iterators.get(i, idx); - LOG.debug("bulk row string " + rowStr + " region " + region); + LOG.debug("bulk row string {} region {}", rowStr, region); return region; } @@ -1733,4 +1586,15 @@ private static void ensureTableEnabled(Admin admin, TableName tableName) throws } } } + + public enum Order { + /** + * Old backups first, most recents last. I.e. sorted by ascending backupId. + */ + OLD_TO_NEW, + /** + * New backups first, oldest last. I.e. sorted by descending backupId. + */ + NEW_TO_OLD + } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java index 7fb7a5768805..ab18c3020564 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java @@ -25,10 +25,16 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupCopyJob; import org.apache.hadoop.hbase.backup.BackupInfo; @@ -41,7 +47,9 @@ import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -136,17 +144,7 @@ public void execute() throws IOException { try (Admin admin = conn.getAdmin()) { // Begin BACKUP beginBackup(backupManager, backupInfo); - String savedStartCode; - boolean firstBackup; - // do snapshot for full table backup - - savedStartCode = backupManager.readBackupStartCode(); - firstBackup = savedStartCode == null || Long.parseLong(savedStartCode) == 0L; - if (firstBackup) { - // This is our first backup. Let's put some marker to system table so that we can hold the - // logs while we do the backup. - backupManager.writeBackupStartCode(0L); - } + // We roll log here before we do the snapshot. It is possible there is duplicate data // in the log that is already in the snapshot. But if we do it after the snapshot, we // could have data loss. @@ -158,13 +156,14 @@ public void execute() throws IOException { // will be part of the snapshot being taken). We gather this list before taking the actual // snapshots for the same reason as the log rolls. List bulkLoadsToDelete = backupManager.readBulkloadRows(tableList); + Map previousLogRollsByHost = backupManager.readRegionServerLastLogRollResult(); Map props = new HashMap<>(); props.put("backupRoot", backupInfo.getBackupRootDir()); admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); - newTimestamps = backupManager.readRegionServerLastLogRollResult(); + Map latestLogRollsByHost = backupManager.readRegionServerLastLogRollResult(); // SNAPSHOT_TABLES: backupInfo.setPhase(BackupPhase.SNAPSHOT); @@ -187,6 +186,50 @@ public void execute() throws IOException { // set overall backup status: complete. Here we make sure to complete the backup. // After this checkpoint, even if entering cancel process, will let the backup finished backupInfo.setState(BackupState.COMPLETE); + + // Scan oldlogs for dead/decommissioned hosts and add their max WAL timestamps + // to newTimestamps. This ensures subsequent incremental backups won't try to back up + // WALs that are already covered by this full backup's snapshot. + Path walRootDir = CommonFSUtils.getWALRootDir(conf); + Path logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME); + Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME); + FileSystem fs = walRootDir.getFileSystem(conf); + + List allLogs = new ArrayList<>(); + for (FileStatus hostLogDir : fs.listStatus(logDir)) { + String host = BackupUtils.parseHostNameFromLogFile(hostLogDir.getPath()); + if (host == null) { + continue; + } + allLogs.addAll(Arrays.asList(fs.listStatus(hostLogDir.getPath()))); + } + allLogs.addAll(Arrays.asList(fs.listStatus(oldLogDir))); + + newTimestamps = new HashMap<>(); + + for (FileStatus log : allLogs) { + if (AbstractFSWALProvider.isMetaFile(log.getPath())) { + continue; + } + String host = BackupUtils.parseHostNameFromLogFile(log.getPath()); + if (host == null) { + continue; + } + long timestamp = BackupUtils.getCreationTime(log.getPath()); + Long previousLogRoll = previousLogRollsByHost.get(host); + Long latestLogRoll = latestLogRollsByHost.get(host); + boolean isInactive = latestLogRoll == null || latestLogRoll.equals(previousLogRoll); + + if (isInactive) { + long currentTs = newTimestamps.getOrDefault(host, 0L); + if (timestamp > currentTs) { + newTimestamps.put(host, timestamp); + } + } else { + newTimestamps.put(host, latestLogRoll); + } + } + // The table list in backupInfo is good for both full backup and incremental backup. // For incremental backup, it contains the incremental backup table set. backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); @@ -194,15 +237,11 @@ public void execute() throws IOException { Map> newTableSetTimestampMap = backupManager.readLogTimestampMap(); - backupInfo.setTableSetTimestampMap(newTableSetTimestampMap); - Long newStartCode = - BackupUtils.getMinValue(BackupUtils.getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); - backupManager.deleteBulkLoadedRows( bulkLoadsToDelete.stream().map(BulkLoad::getRowKey).collect(Collectors.toList())); // backup complete + backupInfo.setTableSetTimestampMap(newTableSetTimestampMap); completeBackup(conn, backupInfo, BackupType.FULL, conf); } catch (Exception e) { failBackup(conn, backupInfo, backupManager, e, "Unexpected BackupException : ", @@ -217,8 +256,20 @@ protected void snapshotTable(Admin admin, TableName tableName, String snapshotNa int pause = conf.getInt(BACKUP_ATTEMPTS_PAUSE_MS_KEY, DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS); int attempts = 0; + Pattern regex = Pattern.compile(snapshotName); + while (attempts++ < maxAttempts) { try { + if ( + admin.listSnapshots(regex).stream() + .anyMatch(snapshot -> snapshot.getName().equals(snapshotName)) + ) { + // If a snapshot takes a long time to run, we may get a TimeoutException from the + // admin, in that case re-attempts to take a snapshot will always fail b/c we'll + // eventually + // attempt to take a snapshot with a name that already exists + return; + } admin.snapshot(snapshotName, tableName); return; } catch (IOException ee) { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index c92c0747e83c..4624e4c5f444 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,7 +29,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; @@ -61,24 +61,11 @@ public IncrementalBackupManager(Connection conn, Configuration conf) throws IOEx */ public Map getIncrBackupLogFileMap() throws IOException { List logList; - Map newTimestamps; - Map previousTimestampMins; + Map previousTimestampMins = + BackupUtils.getRSLogTimestampMins(readLogTimestampMap()); - String savedStartCode = readBackupStartCode(); - - // key: tableName - // value: - Map> previousTimestampMap = readLogTimestampMap(); - - previousTimestampMins = BackupUtils.getRSLogTimestampMins(previousTimestampMap); - - if (LOG.isDebugEnabled()) { - LOG.debug("StartCode " + savedStartCode + "for backupID " + backupInfo.getBackupId()); - } // get all new log files from .logs and .oldlogs after last TS and before new timestamp - if ( - savedStartCode == null || previousTimestampMins == null || previousTimestampMins.isEmpty() - ) { + if (previousTimestampMins.isEmpty()) { throw new IOException("Cannot read any previous back up timestamps from backup system table. " + "In order to create an incremental backup, at least one full backup is needed."); } @@ -91,12 +78,48 @@ public Map getIncrBackupLogFileMap() throws IOException { admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); } - newTimestamps = readRegionServerLastLogRollResult(); + Map newTimestamps = readRegionServerLastLogRollResult(); + + Map latestLogRollByHost = readRegionServerLastLogRollResult(); + for (Map.Entry entry : latestLogRollByHost.entrySet()) { + String host = entry.getKey(); + long latestLogRoll = entry.getValue(); + Long earliestTimestampToIncludeInBackup = previousTimestampMins.get(host); + + boolean isInactive = earliestTimestampToIncludeInBackup != null + && earliestTimestampToIncludeInBackup > latestLogRoll; + + long latestTimestampToIncludeInBackup; + if (isInactive) { + LOG.debug("Avoided resetting latest timestamp boundary for {} from {} to {}", host, + earliestTimestampToIncludeInBackup, latestLogRoll); + latestTimestampToIncludeInBackup = earliestTimestampToIncludeInBackup; + } else { + latestTimestampToIncludeInBackup = latestLogRoll; + } + newTimestamps.put(host, latestTimestampToIncludeInBackup); + } - logList = getLogFilesForNewBackup(previousTimestampMins, newTimestamps, conf, savedStartCode); + logList = getLogFilesForNewBackup(previousTimestampMins, newTimestamps, conf); logList = excludeProcV2WALs(logList); backupInfo.setIncrBackupFileList(logList); + // Update boundaries based on WALs that will be backed up + for (String logFile : logList) { + Path logPath = new Path(logFile); + String logHost = BackupUtils.parseHostFromOldLog(logPath); + if (logHost == null) { + logHost = BackupUtils.parseHostNameFromLogFile(logPath.getParent()); + } + if (logHost != null) { + long logTs = BackupUtils.getCreationTime(logPath); + Long latestTimestampToIncludeInBackup = newTimestamps.get(logHost); + if (latestTimestampToIncludeInBackup == null || logTs > latestTimestampToIncludeInBackup) { + LOG.info("Updating backup boundary for inactive host {}: timestamp={}", logHost, logTs); + newTimestamps.put(logHost, logTs); + } + } + } return newTimestamps; } @@ -121,16 +144,15 @@ private List excludeProcV2WALs(List logList) { * @param olderTimestamps the timestamp for each region server of the last backup. * @param newestTimestamps the timestamp for each region server that the backup should lead to. * @param conf the Hadoop and Hbase configuration - * @param savedStartCode the startcode (timestamp) of last successful backup. * @return a list of log files to be backed up * @throws IOException exception */ private List getLogFilesForNewBackup(Map olderTimestamps, - Map newestTimestamps, Configuration conf, String savedStartCode) - throws IOException { + Map newestTimestamps, Configuration conf) throws IOException { LOG.debug("In getLogFilesForNewBackup()\n" + "olderTimestamps: " + olderTimestamps + "\n newestTimestamps: " + newestTimestamps); + long prevBackupStartTs = Collections.min(olderTimestamps.values()); Path walRootDir = CommonFSUtils.getWALRootDir(conf); Path logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME); Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME); @@ -227,7 +249,7 @@ private List getLogFilesForNewBackup(Map olderTimestamps, * our last backup. */ if (oldTimeStamp == null) { - if (currentLogTS < Long.parseLong(savedStartCode)) { + if (currentLogTS < prevBackupStartTs) { // This log file is really old, its region server was before our last backup. continue; } else { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 1e07c026f0aa..38a0847aa0a8 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -47,13 +47,15 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; import org.apache.hadoop.hbase.mapreduce.WALPlayer; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotManifest; import org.apache.hadoop.hbase.snapshot.SnapshotRegionLocator; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.util.Tool; @@ -63,6 +65,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; /** @@ -197,6 +200,9 @@ private void mergeSplitAndCopyBulkloadedHFiles(List activeFiles, int numActiveFiles = activeFiles.size(); updateFileLists(activeFiles, archiveFiles); if (activeFiles.size() < numActiveFiles) { + // We've archived some files, delete bulkloads directory + // and re-try + deleteBulkLoadDirectory(); continue; } @@ -239,7 +245,7 @@ private void mergeSplitAndCopyBulkloadedHFiles(List files, TableName tn, incrementalCopyBulkloadHFiles(tgtFs, tn); } - private void updateFileLists(List activeFiles, List archiveFiles) + public void updateFileLists(List activeFiles, List archiveFiles) throws IOException { List newlyArchived = new ArrayList<>(); @@ -249,9 +255,24 @@ private void updateFileLists(List activeFiles, List archiveFiles } } - if (newlyArchived.size() > 0) { + if (!newlyArchived.isEmpty()) { + String rootDir = CommonFSUtils.getRootDir(conf).toString(); + activeFiles.removeAll(newlyArchived); - archiveFiles.addAll(newlyArchived); + for (String file : newlyArchived) { + String archivedFile = file.substring(rootDir.length() + 1); + Path archivedFilePath = new Path(HFileArchiveUtil.getArchivePath(conf), archivedFile); + archivedFile = archivedFilePath.toString(); + + if (!fs.exists(archivedFilePath)) { + throw new IOException( + String.format("File %s not longer exists, and no archived file %s exists for it", file, + archivedFile)); + } + + LOG.debug("Archived file {} has been updated", archivedFile); + archiveFiles.add(archivedFile); + } } LOG.debug(newlyArchived.size() + " files have been archived."); @@ -284,8 +305,7 @@ public void execute() throws IOException, ColumnFamilyMismatchException { // case INCREMENTAL_COPY: try { - // copy out the table and region info files for each table - BackupUtils.copyTableRegionInfo(conn, backupInfo, conf); + BackupUtils.copyTableDescriptor(conn, backupInfo, conf); setupRegionLocator(); // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT convertWALsToHFiles(); @@ -312,14 +332,10 @@ public void execute() throws IOException, ColumnFamilyMismatchException { Map> newTableSetTimestampMap = backupManager.readLogTimestampMap(); - backupInfo.setTableSetTimestampMap(newTableSetTimestampMap); - Long newStartCode = - BackupUtils.getMinValue(BackupUtils.getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); - List bulkLoads = handleBulkLoad(backupInfo.getTableNames()); // backup complete + backupInfo.setTableSetTimestampMap(newTableSetTimestampMap); completeBackup(conn, backupInfo, BackupType.INCREMENTAL, conf); List bulkLoadedRows = Lists.transform(bulkLoads, BulkLoad::getRowKey); @@ -332,6 +348,7 @@ public void execute() throws IOException, ColumnFamilyMismatchException { } protected void incrementalCopyHFiles(String[] files, String backupDest) throws IOException { + boolean diskBasedSortingOriginalValue = HFileOutputFormat2.diskBasedSortingEnabled(conf); try { LOG.debug("Incremental copy HFiles is starting. dest=" + backupDest); // set overall backup phase: incremental_copy @@ -346,6 +363,7 @@ protected void incrementalCopyHFiles(String[] files, String backupDest) throws I LOG.debug("Setting incremental copy HFiles job name to : " + jobname); } conf.set(JOB_NAME_CONF_KEY, jobname); + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); BackupCopyJob copyService = BackupRestoreFactory.getBackupCopyJob(conf); int res = copyService.copy(backupInfo, backupManager, conf, BackupType.INCREMENTAL, strArr); @@ -358,6 +376,8 @@ protected void incrementalCopyHFiles(String[] files, String backupDest) throws I + " finished."); } finally { deleteBulkLoadDirectory(); + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + diskBasedSortingOriginalValue); } } @@ -411,7 +431,16 @@ protected void walToHFiles(List dirPaths, List tableList) throws conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";"); conf.setBoolean(HFileOutputFormat2.TABLE_NAME_WITH_NAMESPACE_INCLUSIVE_KEY, true); conf.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); + + conf.setBoolean(HFileOutputFormat2.SET_MAX_SEQ_ID_KEY, true); + conf.set(JOB_NAME_CONF_KEY, jobname); + + boolean diskBasedSortingEnabledOriginalValue = HFileOutputFormat2.diskBasedSortingEnabled(conf); + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); + + // Rack-aware WAL processing configuration is set directly via command line to the same key + // WALPlayer uses String[] playerArgs = { dirs, StringUtils.join(tableList, ",") }; try { @@ -427,6 +456,11 @@ protected void walToHFiles(List dirPaths, List tableList) throws } catch (Exception ee) { throw new IOException("Can not convert from directory " + dirs + " (check Hadoop, HBase and WALPlayer M/R job logs) ", ee); + } finally { + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + diskBasedSortingEnabledOriginalValue); + conf.unset(WALPlayer.INPUT_FILES_SEPARATOR_KEY); + conf.unset(JOB_NAME_CONF_KEY); } } @@ -441,7 +475,7 @@ private void incrementalCopyBulkloadHFiles(FileSystem tgtFs, TableName tn) throw List files = new ArrayList<>(); while (locatedFiles.hasNext()) { LocatedFileStatus file = locatedFiles.next(); - if (file.isFile() && HFile.isHFileFormat(tgtFs, file.getPath())) { + if (file.isFile() && StoreFileInfo.isHFile(file.getPath())) { files.add(file.getPath().toString()); } } @@ -541,6 +575,13 @@ private void verifyCfCompatibility(Set tables, SnapshotDescriptionUtils.readSnapshotInfo(fs, manifestDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, manifestDir, snapshotDescription); + if ( + SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(), + snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()) + ) { + throw new SnapshotTTLExpiredException( + ProtobufUtil.createSnapshotDesc(snapshotDescription)); + } ColumnFamilyDescriptor[] backupCfs = manifest.getTableDescriptor().getColumnFamilies(); if (!areCfsCompatible(currentCfs, backupCfs)) { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java index 30c27f01faaf..5d6d3242b120 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; + import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -310,7 +312,8 @@ protected List getAncestors(BackupInfo backupInfo) throws IOExcepti Set tablesToCover = new HashSet<>(backupInfo.getTables()); // Go over the backup history list from newest to oldest - List allHistoryList = backupManager.getBackupHistory(true); + List allHistoryList = + backupManager.getBackupHistory(withState(BackupState.COMPLETE)); for (BackupInfo backup : allHistoryList) { // If the image has a different rootDir, it cannot be an ancestor. if (!Objects.equals(backup.getBackupRootDir(), backupInfo.getBackupRootDir())) { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java index 28db0c605f79..2eaa63a3961a 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; @@ -33,11 +34,14 @@ import org.apache.hadoop.hbase.mapreduce.CellSortReducer; import org.apache.hadoop.hbase.mapreduce.HFileInputFormat; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; +import org.apache.hadoop.hbase.mapreduce.KeyOnlyCellComparable; +import org.apache.hadoop.hbase.mapreduce.PreSortedCellsReducer; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.snapshot.SnapshotRegionLocator; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.MapReduceExtendedCell; import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; @@ -71,18 +75,28 @@ protected MapReduceHFileSplitterJob(final Configuration c) { /** * A mapper that just writes out cells. This one can be used together with {@link CellSortReducer} */ - static class HFileCellMapper extends Mapper { + static class HFileCellMapper extends Mapper, Cell> { + + private boolean diskBasedSortingEnabled = false; @Override public void map(NullWritable key, Cell value, Context context) throws IOException, InterruptedException { - context.write(new ImmutableBytesWritable(CellUtil.cloneRow(value)), - new MapReduceExtendedCell(value)); + ExtendedCell extendedCell = (ExtendedCell) value; + context.write(wrap(extendedCell), new MapReduceExtendedCell(extendedCell)); } @Override public void setup(Context context) throws IOException { - // do nothing + diskBasedSortingEnabled = + HFileOutputFormat2.diskBasedSortingEnabled(context.getConfiguration()); + } + + private WritableComparable wrap(ExtendedCell cell) { + if (diskBasedSortingEnabled) { + return new KeyOnlyCellComparable(cell); + } + return new ImmutableBytesWritable(CellUtil.cloneRow(cell)); } } @@ -105,14 +119,28 @@ public Job createSubmittableJob(String[] args) throws IOException { job.getConfiguration().setBoolean(HFileOutputFormat2.EXTENDED_CELL_SERIALIZATION_ENABLED_KEY, true); job.setJarByClass(MapReduceHFileSplitterJob.class); + + // Use standard HFileInputFormat which now supports location resolver automatically + // HFileInputFormat will automatically detect and log rack-awareness configuration job.setInputFormatClass(HFileInputFormat.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); + boolean diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); + if (diskBasedSortingEnabled) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + } else { + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + } if (hfileOutPath != null) { LOG.debug("add incremental job :" + hfileOutPath + " from " + inputDirs); TableName tableName = TableName.valueOf(tabName); job.setMapperClass(HFileCellMapper.class); - job.setReducerClass(CellSortReducer.class); + if (diskBasedSortingEnabled) { + job.setReducerClass(PreSortedCellsReducer.class); + } else { + job.setReducerClass(CellSortReducer.class); + } Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); job.setMapOutputValueClass(MapReduceExtendedCell.class); @@ -147,6 +175,11 @@ private void usage(final String errorMsg) { System.err.println("Other options:"); System.err.println(" -D " + JOB_NAME_CONF_KEY + "=jobName - use the specified mapreduce job name for the HFile splitter"); + + System.err.println("Rack-aware processing option:"); + System.err.println(" -D" + HFileInputFormat.CONF_HFILE_LOCATION_RESOLVER_CLASS + "= - " + + "HFile location resolver class for rack-aware processing"); + System.err.println("For performance also consider the following options:\n" + " -Dmapreduce.map.speculative=false\n" + " -Dmapreduce.reduce.speculative=false"); } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java index 6bf1edc7bf03..45fe961ac25b 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hbase.backup.master; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,11 +30,12 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.backup.BackupRestoreConstants; import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.backup.util.BackupBoundaries; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.master.HMaster; @@ -41,7 +44,6 @@ import org.apache.hadoop.hbase.master.region.MasterRegionFactory; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; -import org.apache.hadoop.hbase.wal.WAL; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +58,8 @@ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) public class BackupLogCleaner extends BaseLogCleanerDelegate { private static final Logger LOG = LoggerFactory.getLogger(BackupLogCleaner.class); + private static final long TS_BUFFER_DEFAULT = Duration.ofHours(1).toMillis(); + static final String TS_BUFFER_KEY = "hbase.backup.log.cleaner.timestamp.buffer.ms"; private boolean stopped = false; private Connection conn; @@ -85,9 +89,11 @@ public void init(Map params) { * Calculates the timestamp boundary up to which all backup roots have already included the WAL. * I.e. WALs with a lower (= older) or equal timestamp are no longer needed for future incremental * backups. + * @param backups all completed or running backups to use for the calculation of the boundary + * @param tsBuffer a buffer (in ms) to lower the boundary for the default bound */ - private Map serverToPreservationBoundaryTs(List backups) - throws IOException { + protected static BackupBoundaries calculatePreservationBoundary(List backups, + long tsBuffer) { if (LOG.isDebugEnabled()) { LOG.debug( "Cleaning WALs if they are older than the WAL cleanup time-boundary. " @@ -96,11 +102,12 @@ private Map serverToPreservationBoundaryTs(List backu backups.stream().map(BackupInfo::getBackupId).sorted().collect(Collectors.joining(", "))); } - // This map tracks, for every backup root, the most recent created backup (= highest timestamp) + // This map tracks, for every backup root, the most recent (= highest timestamp) completed + // backup, or if there is no such one, the currently running backup (if any) Map newestBackupPerRootDir = new HashMap<>(); for (BackupInfo backup : backups) { BackupInfo existingEntry = newestBackupPerRootDir.get(backup.getBackupRootDir()); - if (existingEntry == null || existingEntry.getStartTs() < backup.getStartTs()) { + if (existingEntry == null || existingEntry.getState() == BackupState.RUNNING) { newestBackupPerRootDir.put(backup.getBackupRootDir(), backup); } } @@ -112,27 +119,13 @@ private Map serverToPreservationBoundaryTs(List backu .collect(Collectors.joining(", "))); } - // This map tracks, for every RegionServer, the least recent (= oldest / lowest timestamp) - // inclusion in any backup. In other words, it is the timestamp boundary up to which all backup - // roots have included the WAL in their backup. - Map boundaries = new HashMap<>(); - for (BackupInfo backupInfo : newestBackupPerRootDir.values()) { - // Iterate over all tables in the timestamp map, which contains all tables covered in the - // backup root, not just the tables included in that specific backup (which could be a subset) - for (TableName table : backupInfo.getTableSetTimestampMap().keySet()) { - for (Map.Entry entry : backupInfo.getTableSetTimestampMap().get(table) - .entrySet()) { - Address address = Address.fromString(entry.getKey()); - Long storedTs = boundaries.get(address); - if (storedTs == null || entry.getValue() < storedTs) { - boundaries.put(address, entry.getValue()); - } - } - } - } + BackupBoundaries.BackupBoundariesBuilder builder = BackupBoundaries.builder(tsBuffer); + newestBackupPerRootDir.values().forEach(builder::update); + BackupBoundaries boundaries = builder.build(); if (LOG.isDebugEnabled()) { - for (Map.Entry entry : boundaries.entrySet()) { + LOG.debug("Boundaries defaultBoundary: {}", boundaries.getDefaultBoundary()); + for (Map.Entry entry : boundaries.getBoundaries().entrySet()) { LOG.debug("Server: {}, WAL cleanup boundary: {}", entry.getKey().getHostName(), entry.getValue()); } @@ -153,19 +146,19 @@ public Iterable getDeletableFiles(Iterable files) { return files; } - Map serverToPreservationBoundaryTs; - try { - try (BackupManager backupManager = new BackupManager(conn, getConf())) { - serverToPreservationBoundaryTs = - serverToPreservationBoundaryTs(backupManager.getBackupHistory(true)); - } + BackupBoundaries boundaries; + try (BackupSystemTable sysTable = new BackupSystemTable(conn)) { + long tsBuffer = getConf().getLong(TS_BUFFER_KEY, TS_BUFFER_DEFAULT); + List backupHistory = sysTable.getBackupHistory( + i -> EnumSet.of(BackupState.COMPLETE, BackupState.RUNNING).contains(i.getState())); + boundaries = calculatePreservationBoundary(backupHistory, tsBuffer); } catch (IOException ex) { LOG.error("Failed to analyse backup history with exception: {}. Retaining all logs", ex.getMessage(), ex); return Collections.emptyList(); } for (FileStatus file : files) { - if (canDeleteFile(serverToPreservationBoundaryTs, file.getPath())) { + if (canDeleteFile(boundaries, file.getPath())) { filteredFiles.add(file); } } @@ -200,54 +193,17 @@ public boolean isStopped() { return this.stopped; } - protected static boolean canDeleteFile(Map addressToBoundaryTs, Path path) { + protected static boolean canDeleteFile(BackupBoundaries boundaries, Path path) { if (isHMasterWAL(path)) { return true; } - - try { - String hostname = BackupUtils.parseHostNameFromLogFile(path); - if (hostname == null) { - LOG.warn( - "Cannot parse hostname from RegionServer WAL file: {}. Ignoring cleanup of this log", - path); - return false; - } - Address walServerAddress = Address.fromString(hostname); - long walTimestamp = WAL.getTimestamp(path.getName()); - - if (!addressToBoundaryTs.containsKey(walServerAddress)) { - if (LOG.isDebugEnabled()) { - LOG.debug("No cleanup WAL time-boundary found for server: {}. Ok to delete file: {}", - walServerAddress.getHostName(), path); - } - return true; - } - - Long backupBoundary = addressToBoundaryTs.get(walServerAddress); - if (backupBoundary >= walTimestamp) { - if (LOG.isDebugEnabled()) { - LOG.debug( - "WAL cleanup time-boundary found for server {}: {}. Ok to delete older file: {}", - walServerAddress.getHostName(), backupBoundary, path); - } - return true; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("WAL cleanup time-boundary found for server {}: {}. Keeping younger file: {}", - walServerAddress.getHostName(), backupBoundary, path); - } - } catch (Exception ex) { - LOG.warn("Error occurred while filtering file: {}. Ignoring cleanup of this log", path, ex); - return false; - } - return false; + return boundaries.isDeletable(path); } private static boolean isHMasterWAL(Path path) { String fn = path.getName(); return fn.startsWith(WALProcedureStore.LOG_PREFIX) - || fn.endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX); + || fn.endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX) + || path.toString().contains(String.format("/%s/", MasterRegionFactory.MASTER_STORE_DIR)); } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java new file mode 100644 index 000000000000..6853fee78154 --- /dev/null +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.util; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks time boundaries for WAL file cleanup during backup operations. Maintains the oldest + * timestamp per RegionServer included in any backup, enabling safe determination of which WAL files + * can be deleted without compromising backup integrity. + */ +@InterfaceAudience.Private +public class BackupBoundaries { + private static final Logger LOG = LoggerFactory.getLogger(BackupBoundaries.class); + + // This map tracks, for every RegionServer, the least recent (= oldest / lowest timestamp) + // inclusion in any backup. In other words, it is the timestamp boundary up to which all backup + // roots have included the WAL in their backup. + private final Map boundaries; + + // The fallback cleanup boundary for RegionServers without explicit backup boundaries + // (e.g., servers that joined after backups began can be checked against this boundary) + private final long defaultBoundary; + + private BackupBoundaries(Map boundaries, long defaultBoundary) { + this.boundaries = boundaries; + this.defaultBoundary = defaultBoundary; + } + + public boolean isDeletable(Path walLogPath) { + try { + String hostname = BackupUtils.parseHostNameFromLogFile(walLogPath); + + if (hostname == null) { + LOG.warn( + "Cannot parse hostname from RegionServer WAL file: {}. Ignoring cleanup of this log", + walLogPath); + return false; + } + + Address address = Address.fromString(hostname); + long pathTs = WAL.getTimestamp(walLogPath.getName()); + + if (!boundaries.containsKey(address)) { + boolean isDeletable = pathTs <= defaultBoundary; + if (LOG.isDebugEnabled()) { + LOG.debug( + "Boundary for {} not found. isDeletable = {} based on defaultBoundary = {} and WAL ts of {}", + walLogPath, isDeletable, defaultBoundary, pathTs); + } + return isDeletable; + } + + long backupTs = boundaries.get(address); + if (pathTs <= backupTs) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "WAL cleanup time-boundary found for server {}: {}. Ok to delete older file: {}", + address.getHostName(), pathTs, walLogPath); + } + return true; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("WAL cleanup time-boundary found for server {}: {}. Keeping younger file: {}", + address.getHostName(), backupTs, walLogPath); + } + + return false; + } catch (Exception e) { + LOG.warn("Error occurred while filtering file: {}. Ignoring cleanup of this log", walLogPath, + e); + return false; + } + } + + public Map getBoundaries() { + return boundaries; + } + + public long getDefaultBoundary() { + return defaultBoundary; + } + + public static BackupBoundariesBuilder builder(long tsCleanupBuffer) { + return new BackupBoundariesBuilder(tsCleanupBuffer); + } + + public static class BackupBoundariesBuilder { + private final Map boundaries = new HashMap<>(); + private final long tsCleanupBuffer; + + private long oldestStartTs = Long.MAX_VALUE; + + private BackupBoundariesBuilder(long tsCleanupBuffer) { + this.tsCleanupBuffer = tsCleanupBuffer; + } + + /** + * Updates the boundaries based on the provided backup info. + * @param backupInfo the most recent completed backup info for a backup root, or if there is no + * such completed backup, the currently running backup. + */ + public void update(BackupInfo backupInfo) { + switch (backupInfo.getState()) { + case COMPLETE: + // If a completed backup exists in the backup root, we want to protect all logs that + // have been created since the log-roll that happened for that backup. + for (TableName table : backupInfo.getTableSetTimestampMap().keySet()) { + for (Map.Entry entry : backupInfo.getTableSetTimestampMap().get(table) + .entrySet()) { + Address regionServerAddress = Address.fromString(entry.getKey()); + Long logRollTs = entry.getValue(); + + Long storedTs = boundaries.get(regionServerAddress); + if (storedTs == null || logRollTs < storedTs) { + boundaries.put(regionServerAddress, logRollTs); + } + } + } + break; + case RUNNING: + // If there is NO completed backup in the backup root, there are no persisted log-roll + // timestamps available yet. But, we still want to protect all files that have been + // created since the start of the currently running backup. + oldestStartTs = Math.min(oldestStartTs, backupInfo.getStartTs()); + break; + default: + throw new IllegalStateException("Unexpected backupInfo state: " + backupInfo.getState()); + } + } + + public BackupBoundaries build() { + if (boundaries.isEmpty()) { + long defaultBoundary = oldestStartTs - tsCleanupBuffer; + return new BackupBoundaries(Collections.emptyMap(), defaultBoundary); + } + + long oldestRollTs = Collections.min(boundaries.values()); + long defaultBoundary = Math.min(oldestRollTs, oldestStartTs) - tsCleanupBuffer; + return new BackupBoundaries(boundaries, defaultBoundary); + } + } +} diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java index bb3d80a5c1bd..dd6c4a7a3715 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java @@ -24,12 +24,13 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -40,7 +41,6 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; @@ -89,41 +89,19 @@ private BackupUtils() { */ public static Map getRSLogTimestampMins(Map> rsLogTimestampMap) { - if (rsLogTimestampMap == null || rsLogTimestampMap.isEmpty()) { - return null; - } - - HashMap rsLogTimestampMins = new HashMap<>(); - HashMap> rsLogTimestampMapByRS = new HashMap<>(); - - for (Entry> tableEntry : rsLogTimestampMap.entrySet()) { - TableName table = tableEntry.getKey(); - Map rsLogTimestamp = tableEntry.getValue(); - for (Entry rsEntry : rsLogTimestamp.entrySet()) { - String rs = rsEntry.getKey(); - Long ts = rsEntry.getValue(); - rsLogTimestampMapByRS.putIfAbsent(rs, new HashMap<>()); - rsLogTimestampMapByRS.get(rs).put(table, ts); - } - } - - for (Entry> entry : rsLogTimestampMapByRS.entrySet()) { - String rs = entry.getKey(); - rsLogTimestampMins.put(rs, BackupUtils.getMinValue(entry.getValue())); - } - - return rsLogTimestampMins; + return rsLogTimestampMap.values().stream().flatMap(map -> map.entrySet().stream()) + .collect(Collectors.toMap(Entry::getKey, Entry::getValue, Math::min)); } /** - * copy out Table RegionInfo into incremental backup image need to consider move this logic into + * copy out Table descriptor into incremental backup image need to consider move this logic into * HBackupFileSystem * @param conn connection * @param backupInfo backup info * @param conf configuration * @throws IOException exception */ - public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, Configuration conf) + public static void copyTableDescriptor(Connection conn, BackupInfo backupInfo, Configuration conf) throws IOException { Path rootDir = CommonFSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); @@ -147,16 +125,6 @@ public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, C LOG.debug("Attempting to copy table info for:" + table + " target: " + target + " descriptor: " + orig); LOG.debug("Finished copying tableinfo."); - List regions = MetaTableAccessor.getTableRegions(conn, table); - // For each region, write the region info to disk - LOG.debug("Starting to write region info for table " + table); - for (RegionInfo regionInfo : regions) { - Path regionDir = FSUtils - .getRegionDirFromTableDir(new Path(backupInfo.getTableBackupDir(table)), regionInfo); - regionDir = new Path(backupInfo.getTableBackupDir(table), regionDir.getName()); - writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo); - } - LOG.debug("Finished writing region info for table " + table); } } } @@ -339,22 +307,6 @@ public static void checkTargetDir(String backupRootPath, Configuration conf) thr } } - /** - * Get the min value for all the Values a map. - * @param map map - * @return the min value - */ - public static Long getMinValue(Map map) { - Long minTimestamp = null; - if (map != null) { - ArrayList timestampList = new ArrayList<>(map.values()); - Collections.sort(timestampList); - // The min among all the RS log timestamps will be kept in backup system table table. - minTimestamp = timestampList.get(0); - } - return minTimestamp; - } - /** * Parses host name:port from archived WAL path * @param p path @@ -490,24 +442,6 @@ public static String getTableBackupDir(String backupRootDir, String backupId, + Path.SEPARATOR; } - /** - * Sort history list by start time in descending order. - * @param historyList history list - * @return sorted list of BackupCompleteData - */ - public static ArrayList sortHistoryListDesc(ArrayList historyList) { - ArrayList list = new ArrayList<>(); - TreeMap map = new TreeMap<>(); - for (BackupInfo h : historyList) { - map.put(Long.toString(h.getStartTs()), h); - } - Iterator i = map.descendingKeySet().iterator(); - while (i.hasNext()) { - list.add(map.get(i.next())); - } - return list; - } - /** * Calls fs.listStatus() and treats FileNotFoundException as non-fatal This accommodates * differences between hadoop versions, where hadoop 1 does not throw a FileNotFoundException, and @@ -561,12 +495,21 @@ public static String getLogBackupDir(String backupRootDir, String backupId) { + HConstants.HREGION_LOGDIR_NAME; } + /** + * Loads all backup history as stored in files on the given backup root path. + * @return all backup history, from newest (most recent) to oldest (least recent) + */ private static List getHistory(Configuration conf, Path backupRootPath) throws IOException { // Get all (n) history from backup root destination FileSystem fs = FileSystem.get(backupRootPath.toUri(), conf); - RemoteIterator it = fs.listLocatedStatus(backupRootPath); + RemoteIterator it; + try { + it = fs.listLocatedStatus(backupRootPath); + } catch (FileNotFoundException e) { + return Collections.emptyList(); + } List infos = new ArrayList<>(); while (it.hasNext()) { @@ -585,46 +528,23 @@ private static List getHistory(Configuration conf, Path backupRootPa } } // Sort - Collections.sort(infos, new Comparator() { - @Override - public int compare(BackupInfo o1, BackupInfo o2) { - long ts1 = getTimestamp(o1.getBackupId()); - long ts2 = getTimestamp(o2.getBackupId()); - - if (ts1 == ts2) { - return 0; - } - - return ts1 < ts2 ? 1 : -1; - } - - private long getTimestamp(String backupId) { - return Long.parseLong(Iterators.get(Splitter.on('_').split(backupId).iterator(), 1)); - } - }); + infos.sort(Comparator. naturalOrder().reversed()); return infos; } + /** + * Loads all backup history as stored in files on the given backup root path, and returns the + * first n entries matching all given filters. + * @return (subset of) backup history, from newest (most recent) to oldest (least recent) + */ public static List getHistory(Configuration conf, int n, Path backupRootPath, BackupInfo.Filter... filters) throws IOException { List infos = getHistory(conf, backupRootPath); - List ret = new ArrayList<>(); - for (BackupInfo info : infos) { - if (ret.size() == n) { - break; - } - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(info)) { - passed = false; - break; - } - } - if (passed) { - ret.add(info); - } - } - return ret; + + Predicate combinedPredicate = Stream.of(filters) + .map(filter -> (Predicate) filter).reduce(Predicate::and).orElse(x -> true); + + return infos.stream().filter(combinedPredicate).limit(n).collect(Collectors.toList()); } public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs) @@ -665,7 +585,7 @@ public static RestoreRequest createRestoreRequest(String backupRootDir, String b boolean isKeepOriginalSplits) { RestoreRequest.Builder builder = new RestoreRequest.Builder(); RestoreRequest request = builder.withBackupRootDir(backupRootDir).withBackupId(backupId) - .withCheck(check).withFromTables(fromTables).withToTables(toTables).withOvewrite(isOverwrite) + .withCheck(check).withFromTables(fromTables).withToTables(toTables).withOverwrite(isOverwrite) .withKeepOriginalSplits(isKeepOriginalSplits).build(); return request; } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java index 7549b9a8c69a..50b47565d743 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -54,6 +55,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; /** @@ -265,6 +267,12 @@ TableDescriptor getTableDesc(TableName tableName) throws IOException { Path tableInfoPath = this.getTableInfoPath(tableName); SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc); + if ( + SnapshotDescriptionUtils.isExpiredSnapshot(desc.getTtl(), desc.getCreationTime(), + EnvironmentEdgeManager.currentTime()) + ) { + throw new SnapshotTTLExpiredException(ProtobufUtil.createSnapshotDesc(desc)); + } TableDescriptor tableDescriptor = manifest.getTableDescriptor(); if (!tableDescriptor.getTableName().equals(tableName)) { LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " @@ -310,6 +318,12 @@ private void createAndRestoreTable(Connection conn, TableName tableName, TableNa SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath); SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc); + if ( + SnapshotDescriptionUtils.isExpiredSnapshot(desc.getTtl(), desc.getCreationTime(), + EnvironmentEdgeManager.currentTime()) + ) { + throw new SnapshotTTLExpiredException(ProtobufUtil.createSnapshotDesc(desc)); + } tableDescriptor = manifest.getTableDescriptor(); } else { tableDescriptor = getTableDesc(tableName); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/IncrementalBackupRestoreTestBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/IncrementalBackupRestoreTestBase.java new file mode 100644 index 000000000000..7279db159f96 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/IncrementalBackupRestoreTestBase.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.impl.BackupManifest; +import org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException; +import org.apache.hadoop.hbase.tool.BulkLoadHFiles; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.HFileTestUtil; + +import org.apache.hbase.thirdparty.com.google.common.base.Throwables; + +public class IncrementalBackupRestoreTestBase extends TestBackupBase { + + private static final byte[] BULKLOAD_START_KEY = new byte[] { 0x00 }; + private static final byte[] BULKLOAD_END_KEY = new byte[] { Byte.MAX_VALUE }; + + static { + provider = "multiwal"; + } + + protected void checkThrowsCFMismatch(IOException ex, List tables) { + Throwable cause = Throwables.getRootCause(ex); + assertEquals(cause.getClass(), ColumnFamilyMismatchException.class); + ColumnFamilyMismatchException e = (ColumnFamilyMismatchException) cause; + assertEquals(tables, e.getMismatchedTables()); + } + + protected String takeFullBackup(List tables, BackupAdminImpl backupAdmin) + throws IOException { + return takeFullBackup(tables, backupAdmin, false); + } + + protected String takeFullBackup(List tables, BackupAdminImpl backupAdmin, + boolean noChecksumVerify) throws IOException { + BackupRequest req = + createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR, noChecksumVerify); + String backupId = backupAdmin.backupTables(req); + checkSucceeded(backupId); + return backupId; + } + + protected static Path doBulkload(TableName tn, String regionName, byte[]... fams) + throws IOException { + Path regionDir = createHFiles(tn, regionName, fams); + Map results = + BulkLoadHFiles.create(conf1).bulkLoad(tn, regionDir); + assertFalse(results.isEmpty()); + return regionDir; + } + + private static Path createHFiles(TableName tn, String regionName, byte[]... fams) + throws IOException { + Path rootdir = CommonFSUtils.getRootDir(conf1); + Path regionDir = CommonFSUtils.getRegionDir(rootdir, tn, regionName); + + FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); + fs.mkdirs(rootdir); + + for (byte[] fam : fams) { + Path famDir = new Path(regionDir, Bytes.toString(fam)); + Path hFileDir = new Path(famDir, UUID.randomUUID().toString()); + HFileTestUtil.createHFile(conf1, fs, hFileDir, fam, qualName, BULKLOAD_START_KEY, + BULKLOAD_END_KEY, 1000); + } + + return regionDir; + } + + /** + * Check that backup manifest can be produced for a different root. Users may want to move + * existing backups to a different location. + */ + protected void validateRootPathCanBeOverridden(String originalPath, String backupId) + throws IOException { + String anotherRootDir = "/some/other/root/dir"; + Path anotherPath = new Path(anotherRootDir, backupId); + BackupManifest.BackupImage differentLocationImage = BackupManifest.hydrateRootDir( + HBackupFileSystem.getManifest(conf1, new Path(originalPath), backupId).getBackupImage(), + anotherPath); + assertEquals(differentLocationImage.getRootDir(), anotherRootDir); + for (BackupManifest.BackupImage ancestor : differentLocationImage.getAncestors()) { + assertEquals(anotherRootDir, ancestor.getRootDir()); + } + } + + protected List getBackupFiles() throws IOException { + FileSystem fs = TEST_UTIL.getTestFileSystem(); + RemoteIterator iter = fs.listFiles(new Path(BACKUP_ROOT_DIR), true); + List files = new ArrayList<>(); + + while (iter.hasNext()) { + files.add(iter.next()); + } + + return files; + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 2a0be003cab9..77688d0653e6 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -68,9 +68,9 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WALFactory; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -118,7 +118,7 @@ public IncrementalTableBackupClientForTest(Connection conn, String backupId, super(conn, backupId, request); } - @Before + @BeforeEach public void ensurePreviousBackupTestsAreCleanedUp() throws Exception { // Every operation here may not be necessary for any given test, // some often being no-ops. the goal is to help ensure atomicity @@ -163,8 +163,7 @@ public void execute() throws IOException { LOG.debug("For incremental backup, current table set is " + backupManager.getIncrementalBackupTableSet()); newTimestamps = ((IncrementalBackupManager) backupManager).getIncrBackupLogFileMap(); - // copy out the table and region info files for each table - BackupUtils.copyTableRegionInfo(conn, backupInfo, conf); + BackupUtils.copyTableDescriptor(conn, backupInfo, conf); // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT convertWALsToHFiles(); incrementalCopyHFiles(new String[] { getBulkOutputDir().toString() }, @@ -187,14 +186,11 @@ public void execute() throws IOException { Map> newTableSetTimestampMap = backupManager.readLogTimestampMap(); - Long newStartCode = - BackupUtils.getMinValue(BackupUtils.getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); - handleBulkLoad(backupInfo.getTableNames()); failStageIf(Stage.stage_4); // backup complete + backupInfo.setTableSetTimestampMap(newTableSetTimestampMap); completeBackup(conn, backupInfo, BackupType.INCREMENTAL, conf); } catch (Exception e) { @@ -221,16 +217,7 @@ public void execute() throws IOException { // Begin BACKUP beginBackup(backupManager, backupInfo); failStageIf(Stage.stage_0); - String savedStartCode; - boolean firstBackup; // do snapshot for full table backup - savedStartCode = backupManager.readBackupStartCode(); - firstBackup = savedStartCode == null || Long.parseLong(savedStartCode) == 0L; - if (firstBackup) { - // This is our first backup. Let's put some marker to system table so that we can hold the - // logs while we do the backup. - backupManager.writeBackupStartCode(0L); - } failStageIf(Stage.stage_1); // We roll log here before we do the snapshot. It is possible there is duplicate data // in the log that is already in the snapshot. But if we do it after the snapshot, we @@ -274,11 +261,9 @@ public void execute() throws IOException { Map> newTableSetTimestampMap = backupManager.readLogTimestampMap(); - Long newStartCode = - BackupUtils.getMinValue(BackupUtils.getRSLogTimestampMins(newTableSetTimestampMap)); - backupManager.writeBackupStartCode(newStartCode); failStageIf(Stage.stage_4); // backup complete + backupInfo.setTableSetTimestampMap(newTableSetTimestampMap); completeBackup(conn, backupInfo, BackupType.FULL, conf); } catch (Exception e) { @@ -315,6 +300,8 @@ public static void setUpHelper() throws Exception { // Set MultiWAL (with 2 default WAL files per RS) conf1.set(WALFactory.WAL_PROVIDER, provider); TEST_UTIL.startMiniCluster(); + conf1 = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniMapReduceCluster(); if (useSecondCluster) { conf2 = HBaseConfiguration.create(conf1); @@ -327,9 +314,7 @@ public static void setUpHelper() throws Exception { CommonFSUtils.setWALRootDir(TEST_UTIL2.getConfiguration(), p); TEST_UTIL2.startMiniCluster(); } - conf1 = TEST_UTIL.getConfiguration(); - TEST_UTIL.startMiniMapReduceCluster(); BACKUP_ROOT_DIR = new Path(new Path(TEST_UTIL.getConfiguration().get("fs.defaultFS")), BACKUP_ROOT_DIR) .toString(); @@ -348,7 +333,7 @@ public static void setUpHelper() throws Exception { * Setup Cluster with appropriate configurations before running tests. * @throws Exception if starting the mini cluster or setting up the tables fails */ - @BeforeClass + @BeforeAll public static void setUp() throws Exception { TEST_UTIL = new HBaseTestingUtility(); conf1 = TEST_UTIL.getConfiguration(); @@ -365,7 +350,7 @@ private static void populateFromMasterConfig(Configuration masterConf, Configura } } - @AfterClass + @AfterAll public static void tearDown() throws Exception { try { SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin()); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java index b56a15d3a4a8..b553b77ad248 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java @@ -17,26 +17,22 @@ */ package org.apache.hadoop.hbase.backup; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.io.IOException; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupBoundaryTests extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupBoundaryTests.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupBoundaryTests.class); /** @@ -66,33 +62,39 @@ public void testFullBackupMultipleEmpty() throws Exception { * Verify that full backup fails on a single table that does not exist. * @throws Exception if doing the full backup fails */ - @Test(expected = IOException.class) + @Test public void testFullBackupSingleDNE() throws Exception { - LOG.info("test full backup fails on a single table that does not exist"); - List tables = toList("tabledne"); - fullTableBackup(tables); + assertThrows(IOException.class, () -> { + LOG.info("test full backup fails on a single table that does not exist"); + List tables = toList("tabledne"); + fullTableBackup(tables); + }); } /** * Verify that full backup fails on multiple tables that do not exist. * @throws Exception if doing the full backup fails */ - @Test(expected = IOException.class) + @Test public void testFullBackupMultipleDNE() throws Exception { - LOG.info("test full backup fails on multiple tables that do not exist"); - List tables = toList("table1dne", "table2dne"); - fullTableBackup(tables); + assertThrows(IOException.class, () -> { + LOG.info("test full backup fails on multiple tables that do not exist"); + List tables = toList("table1dne", "table2dne"); + fullTableBackup(tables); + }); } /** * Verify that full backup fails on tableset containing real and fake tables. * @throws Exception if doing the full backup fails */ - @Test(expected = IOException.class) + @Test public void testFullBackupMixExistAndDNE() throws Exception { - LOG.info("create full backup fails on tableset containing real and fake table"); + assertThrows(IOException.class, () -> { + LOG.info("create full backup fails on tableset containing real and fake table"); - List tables = toList(table1.getNameAsString(), "tabledne"); - fullTableBackup(tables); + List tables = toList(table1.getNameAsString(), "tabledne"); + fullTableBackup(tables); + }); } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupCommandLineTool.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupCommandLineTool.java index 77bc9bdb9b2f..76d2b4a31706 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupCommandLineTool.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupCommandLineTool.java @@ -17,28 +17,22 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestBackupCommandLineTool { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupCommandLineTool.class); - private final static String USAGE_DESCRIBE = "Usage: hbase backup describe "; private final static String USAGE_CREATE = "Usage: hbase backup create"; private final static String USAGE_HISTORY = "Usage: hbase backup history"; @@ -50,7 +44,7 @@ public class TestBackupCommandLineTool { Configuration conf; - @Before + @BeforeEach public void setUpBefore() throws Exception { conf = HBaseConfiguration.create(); conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java index 785859c52805..40b179833a34 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java @@ -17,38 +17,32 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.ToolRunner; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupDelete extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupDelete.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupDelete.class); /** @@ -138,7 +132,7 @@ public long currentTime() { assertTrue(ret == 0); } catch (Exception e) { LOG.error("failed", e); - Assert.fail(e.getMessage()); + fail(e.getMessage()); } String output = baos.toString(); LOG.info(baos.toString()); @@ -154,7 +148,7 @@ public long currentTime() { assertTrue(ret == 0); } catch (Exception e) { LOG.error("failed", e); - Assert.fail(e.getMessage()); + fail(e.getMessage()); } output = baos.toString(); LOG.info(baos.toString()); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java index 2798e1a16f0d..00acb1afa929 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteRestore.java @@ -17,33 +17,27 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(MediumTests.class) +@Tag(LargeTests.TAG) public class TestBackupDeleteRestore extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupDeleteRestore.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupDeleteRestore.class); /** diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithFailures.java index bc1b110b672b..3aa7235d2d0c 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithFailures.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithFailures.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -26,7 +26,6 @@ import java.util.Optional; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -42,10 +41,9 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,13 +54,9 @@ * TestBackupSmallTests is where tests that don't require bring machines up/down should go All other * tests should have their own classes and extend this one */ -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupDeleteWithFailures extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupDeleteWithFailures.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupDeleteWithFailures.class); public enum Failure { @@ -117,7 +111,7 @@ public void postDeleteSnapshot(ObserverContext ctx * Setup Cluster with appropriate configurations before running tests. * @throws Exception if starting the mini cluster or setting up the tables fails */ - @BeforeClass + @BeforeAll public static void setUp() throws Exception { TEST_UTIL = new HBaseTestingUtility(); conf1 = TEST_UTIL.getConfiguration(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java index 7ce039fd6668..03f7c9ca07dc 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java @@ -17,34 +17,28 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.backup.impl.BackupCommands; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupDescribe extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupDescribe.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupDescribe.class); /** diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java index 7fba9c02e94e..0e8736e6f685 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Arrays; @@ -28,31 +28,26 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(MediumTests.TAG) public class TestBackupHFileCleaner { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupHFileCleaner.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupHFileCleaner.class); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private final static Configuration conf = TEST_UTIL.getConfiguration(); @@ -64,24 +59,24 @@ public class TestBackupHFileCleaner { private Path root; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); TEST_UTIL.startMiniCluster(1); fs = FileSystem.get(conf); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - @Before + @BeforeEach public void setup() throws IOException { root = TEST_UTIL.getDataTestDirOnTestFS(); } - @After + @AfterEach public void cleanup() { try { fs.delete(root, true); @@ -108,11 +103,11 @@ protected Set fetchFullyBackedUpTables(BackupSystemTable tbl) { Iterable deletable; // The first call will not allow any deletions because of the timestamp mechanism. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Collections.emptySet(), Sets.newHashSet(deletable)); // No bulk loads registered, so all files can be deleted. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Sets.newHashSet(file1, file1Archived, file2, file3), Sets.newHashSet(deletable)); // Register some bulk loads. @@ -125,14 +120,21 @@ protected Set fetchFullyBackedUpTables(BackupSystemTable tbl) { } // File 1 can no longer be deleted, because it is registered as a bulk load. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Sets.newHashSet(file2, file3), Sets.newHashSet(deletable)); } + private Iterable callCleaner(BackupHFileCleaner cleaner, Iterable files) { + cleaner.preClean(); + Iterable deletable = cleaner.getDeletableFiles(files); + cleaner.postClean(); + return deletable; + } + private FileStatus createFile(String fileName) throws IOException { Path file = new Path(root, fileName); fs.createNewFile(file); - assertTrue("Test file not created!", fs.exists(file)); + assertTrue(fs.exists(file), "Test file not created!"); return fs.getFileStatus(file); } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupInfoSerialization.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupInfoSerialization.java new file mode 100644 index 000000000000..cd4d50880fba --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupInfoSerialization.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(SmallTests.TAG) +public class TestBackupInfoSerialization { + + private static BackupInfo newIncrementalBackupInfo() { + return new BackupInfo("backup_1234567890", BackupType.INCREMENTAL, + new TableName[] { TableName.valueOf("t1") }, "/hbase/backup"); + } + + private static BackupInfo newFullBackupInfo() { + return new BackupInfo("backup_1234567890", BackupType.FULL, + new TableName[] { TableName.valueOf("t1") }, "/hbase/backup"); + } + + @Test + public void testIncrBackupFileListSurvivesRoundTrip() throws IOException { + List walFiles = Arrays.asList("/hbase/oldWALs/host1.example.com,16020,1234567890.100", + "/hbase/oldWALs/host2.example.com,16020,1234567890.200"); + + BackupInfo original = newIncrementalBackupInfo(); + original.setIncrBackupFileList(walFiles); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertEquals(walFiles, roundTripped.getIncrBackupFileList()); + } + + @Test + public void testIncrBackupFileListOrderIsPreserved() throws IOException { + List walFiles = Arrays.asList("/hbase/oldWALs/c.example.com,16020,1.300", + "/hbase/oldWALs/a.example.com,16020,1.100", "/hbase/oldWALs/b.example.com,16020,1.200"); + + BackupInfo original = newIncrementalBackupInfo(); + original.setIncrBackupFileList(walFiles); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertEquals(walFiles, roundTripped.getIncrBackupFileList()); + } + + @Test + public void testUnsetIncrBackupFileListRemainsNull() throws IOException { + BackupInfo original = newIncrementalBackupInfo(); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertNull(roundTripped.getIncrBackupFileList()); + } + + @Test + public void testEmptyIncrBackupFileListRemainsNull() throws IOException { + BackupInfo original = newIncrementalBackupInfo(); + original.setIncrBackupFileList(Arrays.asList()); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertNull(roundTripped.getIncrBackupFileList()); + } + + @Test + public void testEqualsDistinguishesIncrBackupFileList() throws IOException { + BackupInfo withFiles = newIncrementalBackupInfo(); + withFiles.setIncrBackupFileList(Arrays.asList("/hbase/oldWALs/host1,16020,1.100")); + + BackupInfo withoutFiles = newIncrementalBackupInfo(); + + assertTrue(!withFiles.equals(withoutFiles)); + } + + @Test + public void testTotalBytesCopiedSurvivesRoundTrip() throws IOException { + BackupInfo original = newIncrementalBackupInfo(); + original.setTotalBytesCopied(9876543210L); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertEquals(9876543210L, roundTripped.getTotalBytesCopied()); + } + + @Test + public void testNoChecksumVerifySurvivesRoundTripWhenTrue() throws IOException { + BackupInfo original = newIncrementalBackupInfo(); + original.setNoChecksumVerify(true); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertTrue(roundTripped.getNoChecksumVerify()); + } + + @Test + public void testNoChecksumVerifySurvivesRoundTripWhenFalse() throws IOException { + BackupInfo original = newIncrementalBackupInfo(); + original.setNoChecksumVerify(false); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertFalse(roundTripped.getNoChecksumVerify()); + } + + @Test + public void testIncrTimestampMapSurvivesRoundTrip() throws IOException { + Map t1Timestamps = new HashMap<>(); + t1Timestamps.put("host1.example.com:16020", 100L); + t1Timestamps.put("host2.example.com:16020", 200L); + Map t2Timestamps = new HashMap<>(); + t2Timestamps.put("host1.example.com:16020", 300L); + + Map> incrTimestampMap = new HashMap<>(); + incrTimestampMap.put(TableName.valueOf("t1"), t1Timestamps); + incrTimestampMap.put(TableName.valueOf("t2"), t2Timestamps); + + BackupInfo original = newIncrementalBackupInfo(); + original.setIncrTimestampMap(incrTimestampMap); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertEquals(incrTimestampMap, roundTripped.getIncrTimestampMap()); + } + + @Test + public void testUnsetIncrTimestampMapRemainsNull() throws IOException { + BackupInfo original = newIncrementalBackupInfo(); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertNull(roundTripped.getIncrTimestampMap()); + } + + @Test + public void testIncrementalBackupRetainsHLogTargetDir() throws IOException { + BackupInfo original = newIncrementalBackupInfo(); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertEquals(original.getHLogTargetDir(), roundTripped.getHLogTargetDir()); + } + + @Test + public void testFullBackupHasNoHLogTargetDir() throws IOException { + BackupInfo original = newFullBackupInfo(); + assertNull(original.getHLogTargetDir()); + + BackupInfo roundTripped = BackupInfo.fromByteArray(original.toByteArray()); + + assertNull(roundTripped.getHLogTargetDir()); + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupManager.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupManager.java index cfbc25cef3ea..79601a3c830b 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupManager.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupManager.java @@ -17,47 +17,41 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.backup.impl.BackupManager; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.util.concurrent.Uninterruptibles; -@Category(MediumTests.class) +@Tag(MediumTests.TAG) public class TestBackupManager { private static final Logger LOG = LoggerFactory.getLogger(TestBackupManager.class); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupManager.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); protected static Configuration conf = UTIL.getConfiguration(); protected static MiniHBaseCluster cluster; protected static Connection conn; protected BackupManager backupManager; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); BackupManager.decorateMasterConfiguration(conf); @@ -66,19 +60,19 @@ public static void setUp() throws Exception { conn = UTIL.getConnection(); } - @AfterClass + @AfterAll public static void tearDown() throws IOException { if (cluster != null) { cluster.shutdown(); } } - @Before + @BeforeEach public void before() throws IOException { backupManager = new BackupManager(conn, conn.getConfiguration()); } - @After + @AfterEach public void after() { backupManager.close(); } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java index 38204f68e31a..47dafab89856 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; import org.apache.hadoop.hbase.backup.util.BackupUtils; @@ -31,22 +31,16 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupMerge extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupMerge.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupMerge.class); @Test @@ -72,13 +66,13 @@ public void TestIncBackupMergeRestore() throws Exception { Table t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS); LOG.debug("writing " + ADD_ROWS + " rows to " + table1); - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS); + assertEquals(NB_ROWS_IN_BATCH + ADD_ROWS, TEST_UTIL.countRows(t1)); t1.close(); LOG.debug("written " + ADD_ROWS + " rows to " + table1); Table t2 = insertIntoTable(conn, table2, famName, 1, ADD_ROWS); - Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + ADD_ROWS); + assertEquals(NB_ROWS_IN_BATCH + ADD_ROWS, TEST_UTIL.countRows(t2)); t2.close(); LOG.debug("written " + ADD_ROWS + " rows to " + table2); @@ -115,12 +109,12 @@ public void TestIncBackupMergeRestore() throws Exception { LOG.debug("After incremental restore: " + hTable.getDescriptor()); int countRows = TEST_UTIL.countRows(hTable, famName); LOG.debug("f1 has " + countRows + " rows"); - Assert.assertEquals(NB_ROWS_IN_BATCH + 2 * ADD_ROWS, countRows); + assertEquals(NB_ROWS_IN_BATCH + 2 * ADD_ROWS, countRows); hTable.close(); hTable = conn.getTable(table2_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 2 * ADD_ROWS); + assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 2 * ADD_ROWS); hTable.close(); admin.close(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java index 36cecd3faf58..74064c1cedb4 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java @@ -17,13 +17,12 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; import org.apache.hadoop.hbase.client.Admin; @@ -33,10 +32,8 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,13 +42,9 @@ /** * Create multiple backups for two tables: table1, table2 then perform 1 delete */ -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupMultipleDeletes extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupMultipleDeletes.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupMultipleDeletes.class); @Test @@ -73,7 +66,7 @@ public void testBackupMultipleDeletes() throws Exception { p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); t1.put(p1); } - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH * 2); + assertEquals(NB_ROWS_IN_BATCH * 2, TEST_UTIL.countRows(t1)); t1.close(); // #3 - incremental backup for table1 tables = Lists.newArrayList(table1); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java new file mode 100644 index 000000000000..10e33380f111 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +/** + * Tests that WAL files from offline/inactive RegionServers are handled correctly during backup. + * Specifically verifies that WALs from an offline RS are: + *

    + *
  1. Backed up once in the first backup after the RS goes offline
    2. + *
  2. NOT re-backed up in subsequent backups
    3. + *
+ */ +@Category(LargeTests.class) +public class TestBackupOfflineRS extends TestBackupBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBackupOfflineRS.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestBackupOfflineRS.class); + + @BeforeClass + public static void setUp() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + conf1 = TEST_UTIL.getConfiguration(); + conf1.setInt("hbase.regionserver.info.port", -1); + autoRestoreOnFailure = true; + useSecondCluster = false; + setUpHelper(); + // Start an additional RS so we have at least 2 + TEST_UTIL.getMiniHBaseCluster().startRegionServer(); + TEST_UTIL.waitTableAvailable(table1); + } + + /** + * Tests that when a full backup is taken while an RS is offline (with WALs in oldlogs), the + * offline host's timestamps are recorded so subsequent incremental backups don't re-include those + * WALs. + */ + @Test + public void testBackupWithOfflineRS() throws Exception { + LOG.info("Starting testFullBackupWithOfflineRS"); + + MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); + List tables = Lists.newArrayList(table1); + + if (cluster.getNumLiveRegionServers() < 2) { + cluster.startRegionServer(); + Thread.sleep(2000); + } + + LOG.info("Inserting data to generate WAL entries"); + try (Connection conn = ConnectionFactory.createConnection(conf1)) { + insertIntoTable(conn, table1, famName, 2, 100); + } + + int rsToStop = 0; + HRegionServer rsBeforeStop = cluster.getRegionServer(rsToStop); + String offlineHost = + rsBeforeStop.getServerName().getHostname() + ":" + rsBeforeStop.getServerName().getPort(); + LOG.info("Stopping RS: {}", offlineHost); + + cluster.stopRegionServer(rsToStop); + // Wait for WALs to be moved to oldlogs + Thread.sleep(5000); + + LOG.info("Taking full backup (with offline RS WALs in oldlogs)"); + String fullBackupId = fullTableBackup(tables); + assertTrue("Full backup should succeed", checkSucceeded(fullBackupId)); + + try (BackupSystemTable sysTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + Map> timestamps = sysTable.readLogTimestampMap(BACKUP_ROOT_DIR); + Map rsTimestamps = timestamps.get(table1); + LOG.info("RS timestamps after full backup: {}", rsTimestamps); + + Long tsAfterFullBackup = rsTimestamps.get(offlineHost); + assertNotNull("Offline host should have timestamp recorded in trslm after full backup", + tsAfterFullBackup); + + LOG.info("Taking incremental backup (should NOT include offline RS WALs)"); + String incrBackupId = incrementalTableBackup(tables); + assertTrue("Incremental backup should succeed", checkSucceeded(incrBackupId)); + + timestamps = sysTable.readLogTimestampMap(BACKUP_ROOT_DIR); + rsTimestamps = timestamps.get(table1); + assertFalse("Offline host should not have a boundary ", + rsTimestamps.containsKey(offlineHost)); + } + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRepair.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRepair.java index 239e1409bdd8..69cfdf159973 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRepair.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRepair.java @@ -17,31 +17,25 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import java.util.Set; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.backup.impl.TableBackupClient; import org.apache.hadoop.hbase.backup.impl.TableBackupClient.Stage; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupRepair extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupRepair.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupRepair.class); @Test diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java new file mode 100644 index 000000000000..b84ff32de59b --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.LogRoller; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +@Tag(LargeTests.TAG) +public class TestBackupRestoreExpiry extends TestBackupBase { + + @BeforeAll + public static void setUp() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + conf1 = TEST_UTIL.getConfiguration(); + conf1.setLong(HConstants.DEFAULT_SNAPSHOT_TTL_CONFIG_KEY, 30); + autoRestoreOnFailure = true; + useSecondCluster = false; + setUpHelper(); + } + + public void ensurePreviousBackupTestsAreCleanedUp() throws Exception { + TEST_UTIL.flush(table1); + TEST_UTIL.flush(table2); + + TEST_UTIL.truncateTable(table1).close(); + TEST_UTIL.truncateTable(table2).close(); + + if (TEST_UTIL.getAdmin().tableExists(table1_restore)) { + TEST_UTIL.flush(table1_restore); + TEST_UTIL.truncateTable(table1_restore).close(); + } + + TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().forEach(rst -> { + try { + LogRoller walRoller = rst.getRegionServer().getWalRoller(); + walRoller.requestRollAll(); + walRoller.waitUntilWalRollFinished(); + } catch (Exception ignored) { + } + }); + + try (Table table = TEST_UTIL.getConnection().getTable(table1)) { + loadTable(table); + } + + try (Table table = TEST_UTIL.getConnection().getTable(table2)) { + loadTable(table); + } + } + + @Test + public void testSequentially() throws Exception { + try { + testRestoreOnExpiredFullBackup(); + } catch (Exception e) { + throw e; + } finally { + ensurePreviousBackupTestsAreCleanedUp(); + } + + try { + testIncrementalBackupOnExpiredFullBackup(); + } catch (Exception e) { + throw e; + } finally { + ensurePreviousBackupTestsAreCleanedUp(); + } + } + + public void testRestoreOnExpiredFullBackup() throws Exception { + byte[] mobFam = Bytes.toBytes("mob"); + + List tables = Lists.newArrayList(table1); + TableDescriptor newTable1Desc = + TableDescriptorBuilder.newBuilder(table1Desc).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(mobFam).setMobEnabled(true).setMobThreshold(5L).build()).build(); + TEST_UTIL.getAdmin().modifyTable(newTable1Desc); + + Connection conn = TEST_UTIL.getConnection(); + BackupAdminImpl backupAdmin = new BackupAdminImpl(conn); + BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); + String fullBackupId = backupAdmin.backupTables(request); + assertTrue(checkSucceeded(fullBackupId)); + + TableName[] fromTables = new TableName[] { table1 }; + TableName[] toTables = new TableName[] { table1_restore }; + + EnvironmentEdgeManager.injectEdge(new EnvironmentEdge() { + // time + 30s + @Override + public long currentTime() { + return System.currentTimeMillis() + (30 * 1000); + } + }); + + assertThrows(SnapshotTTLExpiredException.class, () -> { + backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, fullBackupId, false, + fromTables, toTables, true, true)); + }); + + EnvironmentEdgeManager.reset(); + backupAdmin.close(); + } + + public void testIncrementalBackupOnExpiredFullBackup() throws Exception { + byte[] mobFam = Bytes.toBytes("mob"); + + List tables = Lists.newArrayList(table1); + TableDescriptor newTable1Desc = + TableDescriptorBuilder.newBuilder(table1Desc).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(mobFam).setMobEnabled(true).setMobThreshold(5L).build()).build(); + TEST_UTIL.getAdmin().modifyTable(newTable1Desc); + + Connection conn = TEST_UTIL.getConnection(); + BackupAdminImpl backupAdmin = new BackupAdminImpl(conn); + BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); + String fullBackupId = backupAdmin.backupTables(request); + assertTrue(checkSucceeded(fullBackupId)); + + TableName[] fromTables = new TableName[] { table1 }; + TableName[] toTables = new TableName[] { table1_restore }; + + List preRestoreBackupFiles = getBackupFiles(); + backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, fullBackupId, false, + fromTables, toTables, true, true)); + List postRestoreBackupFiles = getBackupFiles(); + + // Check that the backup files are the same before and after the restore process + assertEquals(postRestoreBackupFiles, preRestoreBackupFiles); + assertEquals(NB_ROWS_IN_BATCH, TEST_UTIL.countRows(table1_restore)); + + int ROWS_TO_ADD = 1_000; + // different IDs so that rows don't overlap + insertIntoTable(conn, table1, famName, 3, ROWS_TO_ADD); + insertIntoTable(conn, table1, mobFam, 4, ROWS_TO_ADD); + + Admin admin = conn.getAdmin(); + List currentRegions = TEST_UTIL.getHBaseCluster().getRegions(table1); + for (HRegion region : currentRegions) { + byte[] name = region.getRegionInfo().getEncodedNameAsBytes(); + admin.splitRegionAsync(name).get(); + } + + TEST_UTIL.waitTableAvailable(table1); + + // Make sure we've split regions + assertNotEquals(currentRegions, TEST_UTIL.getHBaseCluster().getRegions(table1)); + + EnvironmentEdgeManager.injectEdge(new EnvironmentEdge() { + // time + 30s + @Override + public long currentTime() { + return System.currentTimeMillis() + (30 * 1000); + } + }); + + IOException e = assertThrows(IOException.class, () -> { + backupAdmin + .backupTables(createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR)); + }); + assertTrue(e.getCause() instanceof SnapshotTTLExpiredException); + + EnvironmentEdgeManager.reset(); + backupAdmin.close(); + } + + private List getBackupFiles() throws IOException { + FileSystem fs = TEST_UTIL.getTestFileSystem(); + RemoteIterator iter = fs.listFiles(new Path(BACKUP_ROOT_DIR), true); + List files = new ArrayList<>(); + + while (iter.hasNext()) { + files.add(iter.next()); + } + + return files; + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java index 6fb538fe629a..ee89aae91e33 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java @@ -22,8 +22,8 @@ import static org.apache.hadoop.hbase.backup.BackupTestUtil.verifyBackup; import static org.apache.hadoop.hbase.backup.BackupType.FULL; import static org.apache.hadoop.hbase.backup.BackupType.INCREMENTAL; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.time.Instant; @@ -31,12 +31,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; import org.apache.hadoop.hbase.client.Admin; @@ -48,40 +48,37 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testing.TestingHBaseCluster; import org.apache.hadoop.hbase.testing.TestingHBaseClusterOption; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.params.provider.Arguments; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(MediumTests.class) -@RunWith(Parameterized.class) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: restoreToOtherTable={0}") public class TestBackupRestoreOnEmptyEnvironment { private static final Logger LOG = LoggerFactory.getLogger(TestBackupRestoreOnEmptyEnvironment.class); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupRestoreOnEmptyEnvironment.class); + public boolean restoreToOtherTable; - @Parameterized.Parameters(name = "{index}: restoreToOtherTable={0}") - public static Iterable data() { - return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; + public TestBackupRestoreOnEmptyEnvironment(boolean restoreToOtherTable) { + this.restoreToOtherTable = restoreToOtherTable; + } + + public static Stream parameters() { + return Stream.of(Arguments.of(true), Arguments.of(false)); } - @Parameterized.Parameter(0) - public boolean restoreToOtherTable; private TableName sourceTable; private TableName targetTable; @@ -89,7 +86,7 @@ public static Iterable data() { private static Path BACKUP_ROOT_DIR; private static final byte[] COLUMN_FAMILY = Bytes.toBytes("0"); - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { Configuration conf = HBaseConfiguration.create(); enableBackup(conf); @@ -98,12 +95,12 @@ public static void beforeClass() throws Exception { BACKUP_ROOT_DIR = new Path(new Path(conf.get("fs.defaultFS")), new Path("/backupIT")); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { cluster.stop(); } - @Before + @BeforeEach public void setUp() throws Exception { sourceTable = TableName.valueOf("table"); targetTable = TableName.valueOf("another-table"); @@ -111,12 +108,12 @@ public void setUp() throws Exception { createTable(targetTable); } - @After + @AfterEach public void removeTables() throws Exception { deleteTables(); } - @Test + @TestTemplate public void testRestoreToCorrectTable() throws Exception { Instant timestamp = Instant.now().minusSeconds(10); @@ -140,7 +137,7 @@ public void testRestoreToCorrectTable() throws Exception { } - @Test + @TestTemplate public void testRestoreCorrectTableForIncremental() throws Exception { Instant timestamp = Instant.now().minusSeconds(10); @@ -231,7 +228,7 @@ private void restore(String backupId, TableName sourceTableName, TableName targe try (Connection connection = ConnectionFactory.createConnection(cluster.getConf()); BackupAdmin backupAdmin = new BackupAdminImpl(connection)) { RestoreRequest restoreRequest = new RestoreRequest.Builder().withBackupId(backupId) - .withBackupRootDir(BACKUP_ROOT_DIR.toString()).withOvewrite(true) + .withBackupRootDir(BACKUP_ROOT_DIR.toString()).withOverwrite(true) .withFromTables(new TableName[] { sourceTableName }) .withToTables(new TableName[] { targetTableName }).build(); backupAdmin.restore(restoreRequest); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java index 72c4a464e94d..3b20d3624a5c 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java @@ -21,9 +21,9 @@ import static org.apache.hadoop.hbase.backup.BackupTestUtil.enableBackup; import static org.apache.hadoop.hbase.backup.BackupTestUtil.verifyBackup; import static org.apache.hadoop.hbase.backup.BackupType.FULL; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.ByteBuffer; @@ -33,13 +33,13 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; @@ -54,40 +54,36 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testing.TestingHBaseCluster; import org.apache.hadoop.hbase.testing.TestingHBaseClusterOption; import org.apache.hadoop.hbase.tool.BulkLoadHFiles; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.params.provider.Arguments; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(MediumTests.class) -@RunWith(Parameterized.class) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: useBulkLoad={0}") public class TestBackupRestoreWithModifications { private static final Logger LOG = LoggerFactory.getLogger(TestBackupRestoreWithModifications.class); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupRestoreWithModifications.class); + public boolean useBulkLoad; - @Parameterized.Parameters(name = "{index}: useBulkLoad={0}") - public static Iterable data() { - return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; + public TestBackupRestoreWithModifications(boolean useBulkLoad) { + this.useBulkLoad = useBulkLoad; } - @Parameterized.Parameter(0) - public boolean useBulkLoad; + public static Stream parameters() { + return Stream.of(Arguments.of(true), Arguments.of(false)); + } private TableName sourceTable; private TableName targetTable; @@ -97,7 +93,7 @@ public static Iterable data() { private static final Path BACKUP_ROOT_DIR = new Path("backupIT"); private static final byte[] COLUMN_FAMILY = Bytes.toBytes("0"); - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { Configuration conf = HBaseConfiguration.create(); enableBackup(conf); @@ -105,12 +101,12 @@ public static void beforeClass() throws Exception { cluster.start(); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { cluster.stop(); } - @Before + @BeforeEach public void setUp() throws Exception { sourceTable = TableName.valueOf("table-" + useBulkLoad); targetTable = TableName.valueOf("another-table-" + useBulkLoad); @@ -119,7 +115,7 @@ public void setUp() throws Exception { createTable(targetTable); } - @Test + @TestTemplate public void testModificationsOnTable() throws Exception { Instant timestamp = Instant.now(); @@ -228,7 +224,7 @@ private void restore(String backupId, TableName sourceTableName, TableName targe try (Connection connection = ConnectionFactory.createConnection(cluster.getConf()); BackupAdmin backupAdmin = new BackupAdminImpl(connection)) { RestoreRequest restoreRequest = new RestoreRequest.Builder().withBackupId(backupId) - .withBackupRootDir(BACKUP_ROOT_DIR.toString()).withOvewrite(true) + .withBackupRootDir(BACKUP_ROOT_DIR.toString()).withOverwrite(true) .withFromTables(new TableName[] { sourceTableName }) .withToTables(new TableName[] { targetTableName }).build(); backupAdmin.restore(restoreRequest); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java index fa624250929d..0c1870cdc71a 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java @@ -17,125 +17,122 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.List; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupShowHistory extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupShowHistory.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupShowHistory.class); - private boolean findBackup(List history, String backupId) { - assertTrue(history.size() > 0); - boolean success = false; - for (BackupInfo info : history) { - if (info.getBackupId().equals(backupId)) { - success = true; - break; - } - } - return success; - } - /** - * Verify that full backup is created on a single table with data correctly. Verify that history - * works as expected. - * @throws Exception if doing the backup or an operation on the tables fails + * Verify that backup history retrieval works as expected. */ @Test public void testBackupHistory() throws Exception { - LOG.info("test backup history on a single table with data"); - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); + // Test without backups present + List history = getBackupAdmin().getHistory(10); + assertEquals(0, history.size()); + + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR)); + assertEquals(0, history.size()); + + // Create first backup + String backupId = fullTableBackup(Lists.newArrayList(table1)); assertTrue(checkSucceeded(backupId)); LOG.info("backup complete"); - List history = getBackupAdmin().getHistory(10); - assertTrue(findBackup(history, backupId)); - BackupInfo.Filter nullFilter = info -> true; - history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), nullFilter); - assertTrue(findBackup(history, backupId)); + // Tests with one backup present + history = getBackupAdmin().getHistory(10); + assertEquals(1, history.size()); + assertEquals(backupId, history.get(0).getBackupId()); + + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR)); + assertEquals(1, history.size()); + assertEquals(backupId, history.get(0).getBackupId()); + + String output = runHistoryCommand(10); + assertTrue(output.indexOf(backupId) > 0); + // Create second backup + String backupId2 = fullTableBackup(Lists.newArrayList(table2)); + assertTrue(checkSucceeded(backupId2)); + LOG.info("backup complete: " + table2); + + // Test with multiple backups + history = getBackupAdmin().getHistory(10); + assertEquals(2, history.size()); + assertEquals(backupId2, history.get(0).getBackupId()); + assertEquals(backupId, history.get(1).getBackupId()); + + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR)); + assertEquals(2, history.size()); + assertEquals(backupId2, history.get(0).getBackupId()); + assertEquals(backupId, history.get(1).getBackupId()); + + output = runHistoryCommand(10); + int idx1 = output.indexOf(backupId); + int idx2 = output.indexOf(backupId2); + assertTrue(idx1 >= 0); // Backup 1 is listed + assertTrue(idx2 >= 0); // Backup 2 is listed + assertTrue(idx2 < idx1); // Newest backup (Backup 2) comes first + + // Test with multiple backups & n == 1 + history = getBackupAdmin().getHistory(1); + assertEquals(1, history.size()); + assertEquals(backupId2, history.get(0).getBackupId()); + + history = BackupUtils.getHistory(conf1, 1, new Path(BACKUP_ROOT_DIR)); + assertEquals(1, history.size()); + assertEquals(backupId2, history.get(0).getBackupId()); + + output = runHistoryCommand(1); + idx1 = output.indexOf(backupId); + idx2 = output.indexOf(backupId2); + assertTrue(idx2 > 0); // most recent backup is listed + assertEquals(-1, idx1); // second most recent backup isn't listed + + // Test with multiple backups & filtering + BackupInfo.Filter tableNameFilter = i -> i.getTableNames().contains(table1); + + history = getBackupAdmin().getHistory(10, tableNameFilter); + assertEquals(1, history.size()); + assertEquals(backupId, history.get(0).getBackupId()); + + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), tableNameFilter); + assertEquals(1, history.size()); + assertEquals(backupId, history.get(0).getBackupId()); + + } + + private String runHistoryCommand(int n) throws Exception { + String[] args = new String[] { "history", "-n", String.valueOf(n), "-p", BACKUP_ROOT_DIR }; ByteArrayOutputStream baos = new ByteArrayOutputStream(); System.setOut(new PrintStream(baos)); - String[] args = new String[] { "history", "-n", "10", "-p", BACKUP_ROOT_DIR }; - // Run backup + LOG.info("Running history command"); int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - LOG.info("show_history"); + assertEquals(0, ret); + String output = baos.toString(); LOG.info(output); baos.close(); - assertTrue(output.indexOf(backupId) > 0); - - tableList = Lists.newArrayList(table2); - String backupId2 = fullTableBackup(tableList); - assertTrue(checkSucceeded(backupId2)); - LOG.info("backup complete: " + table2); - BackupInfo.Filter tableNameFilter = image -> { - if (table1 == null) { - return true; - } - - List names = image.getTableNames(); - return names.contains(table1); - }; - BackupInfo.Filter tableSetFilter = info -> { - String backupId1 = info.getBackupId(); - return backupId1.startsWith("backup"); - }; - - history = getBackupAdmin().getHistory(10, tableNameFilter, tableSetFilter); - assertTrue(history.size() > 0); - boolean success = true; - for (BackupInfo info : history) { - if (!info.getTableNames().contains(table1)) { - success = false; - break; - } - } - assertTrue(success); - - history = - BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), tableNameFilter, tableSetFilter); - assertTrue(history.size() > 0); - success = true; - for (BackupInfo info : history) { - if (!info.getTableNames().contains(table1)) { - success = false; - break; - } - } - assertTrue(success); - - args = - new String[] { "history", "-n", "10", "-p", BACKUP_ROOT_DIR, "-t", "table1", "-s", "backup" }; - // Run backup - ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - LOG.info("show_history"); + return output; } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSmallTests.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSmallTests.java index 83cc19578ade..361c3428282f 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSmallTests.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSmallTests.java @@ -17,25 +17,21 @@ */ package org.apache.hadoop.hbase.backup; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.io.IOException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(MediumTests.TAG) public class TestBackupSmallTests extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupSmallTests.class); - private static final UserGroupInformation DIANA = UserGroupInformation.createUserForTesting("diana", new String[] {}); private static final String PERMISSION_TEST_PATH = Path.SEPARATOR + "permissionUT"; @@ -47,14 +43,16 @@ public void testBackupPathIsAccessible() throws Exception { fs.mkdirs(path); } - @Test(expected = IOException.class) + @Test public void testBackupPathIsNotAccessible() throws Exception { - Path path = new Path(PERMISSION_TEST_PATH); - FileSystem rootFs = FileSystem.get(TEST_UTIL.getConnection().getConfiguration()); - rootFs.mkdirs(path.getParent()); - rootFs.setPermission(path.getParent(), FsPermission.createImmutable((short) 000)); - FileSystem fs = - DFSTestUtil.getFileSystemAs(DIANA, TEST_UTIL.getConnection().getConfiguration()); - fs.mkdirs(path); + assertThrows(IOException.class, () -> { + Path path = new Path(PERMISSION_TEST_PATH); + FileSystem rootFs = FileSystem.get(TEST_UTIL.getConnection().getConfiguration()); + rootFs.mkdirs(path.getParent()); + rootFs.setPermission(path.getParent(), FsPermission.createImmutable((short) 000)); + FileSystem fs = + DFSTestUtil.getFileSystemAs(DIANA, TEST_UTIL.getConnection().getConfiguration()); + fs.mkdirs(path); + }); } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java index 1a1e5dbf1cc1..ecf2915f8249 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java @@ -17,31 +17,25 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupStatusProgress extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupStatusProgress.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupStatusProgress.class); /** diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java index 9f7f81cb3f38..039669e2a46f 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -34,7 +34,6 @@ import java.util.Set; import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; @@ -43,33 +42,28 @@ import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test cases for backup system table API */ -@Category(MediumTests.class) +@Tag(LargeTests.TAG) public class TestBackupSystemTable { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupSystemTable.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); protected static Configuration conf = UTIL.getConfiguration(); protected static MiniHBaseCluster cluster; protected static Connection conn; protected BackupSystemTable table; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); BackupManager.decorateMasterConfiguration(conf); @@ -78,12 +72,12 @@ public static void setUp() throws Exception { conn = UTIL.getConnection(); } - @Before + @BeforeEach public void before() throws IOException { table = new BackupSystemTable(conn); } - @After + @AfterEach public void after() { if (table != null) { table.close(); @@ -107,15 +101,6 @@ public void testUpdateReadDeleteBackupStatus() throws IOException { cleanBackupTable(); } - @Test - public void testWriteReadBackupStartCode() throws IOException { - long code = 100L; - table.writeBackupStartCode(code, "root"); - String readCode = table.readBackupStartCode("root"); - assertEquals(code, Long.parseLong(readCode)); - cleanBackupTable(); - } - private void cleanBackupTable() throws IOException { Admin admin = UTIL.getAdmin(); admin.disableTable(BackupSystemTable.getTableName(conf)); @@ -504,7 +489,7 @@ private List createBackupInfoList(int size) throws InterruptedExcept return list; } - @AfterClass + @AfterAll public static void tearDown() throws IOException { if (cluster != null) { cluster.shutdown(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java index d4e6900fb28b..c29af366ae97 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.hbase.backup; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.IOException; import java.security.PrivilegedAction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; @@ -33,18 +36,14 @@ import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestBackupUtils { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupUtils.class); + private static final Logger LOG = LoggerFactory.getLogger(TestBackupUtils.class); protected static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -82,7 +81,7 @@ public Path run() { } }); // Make sure the directory is in foo1234's home directory - Assert.assertTrue(bulkOutputDir.toString().startsWith(fooHomeDirectory.toString())); + assertTrue(bulkOutputDir.toString().startsWith(fooHomeDirectory.toString())); } @Test @@ -99,18 +98,18 @@ public void testFilesystemWalHostNameParsing() throws IOException { Path testOldWalPath = new Path(oldLogDir, serverName + BackupUtils.LOGNAME_SEPARATOR + EnvironmentEdgeManager.currentTime()); - Assert.assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port, + assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port, BackupUtils.parseHostFromOldLog(testOldWalPath)); Path testMasterWalPath = new Path(oldLogDir, testOldWalPath.getName() + MasterRegionFactory.ARCHIVED_WAL_SUFFIX); - Assert.assertNull(BackupUtils.parseHostFromOldLog(testMasterWalPath)); + assertNull(BackupUtils.parseHostFromOldLog(testMasterWalPath)); // org.apache.hadoop.hbase.wal.BoundedGroupingStrategy does this Path testOldWalWithRegionGroupingPath = new Path(oldLogDir, serverName + BackupUtils.LOGNAME_SEPARATOR + serverName + BackupUtils.LOGNAME_SEPARATOR + "regiongroup-0" + BackupUtils.LOGNAME_SEPARATOR + EnvironmentEdgeManager.currentTime()); - Assert.assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port, + assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port, BackupUtils.parseHostFromOldLog(testOldWalWithRegionGroupingPath)); } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestDeleteFailedBackupDoesNotCascade.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestDeleteFailedBackupDoesNotCascade.java new file mode 100644 index 000000000000..2a9580004e7e --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestDeleteFailedBackupDoesNotCascade.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +/** + * Verify that deleting a FAILED backup does not cascade-delete subsequent COMPLETE backups. A + * FAILED backup's state was rolled back via snapshot restore, so no subsequent backup depends on + * it. + */ +@Category(LargeTests.class) +public class TestDeleteFailedBackupDoesNotCascade extends TestBackupBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDeleteFailedBackupDoesNotCascade.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestDeleteFailedBackupDoesNotCascade.class); + + @Test + public void testDeleteFailedBackupDoesNotCascadeToCompletedIncremental() throws Exception { + LOG.info("Test that deleting a FAILED backup does not cascade to COMPLETE incrementals"); + + List tableList = Lists.newArrayList(table1); + + // Step 1: Create a full backup + String fullBackupId = fullTableBackup(tableList); + assertTrue(checkSucceeded(fullBackupId)); + LOG.info("Full backup {} succeeded", fullBackupId); + + // Step 2: Insert data so the incremental has something to back up + try (Connection conn = ConnectionFactory.createConnection(conf1); + Table t1 = conn.getTable(table1)) { + for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { + Put p = new Put(Bytes.toBytes("row-incr1-" + i)); + p.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t1.put(p); + } + } + + // Step 3: Create a successful incremental backup + String successIncrBackupId = incrementalTableBackup(tableList); + assertTrue(checkSucceeded(successIncrBackupId)); + LOG.info("Successful incremental backup: {}", successIncrBackupId); + + // Step 4: Directly insert a FAILED backup record into the system table. + // This simulates a backup that failed and was recorded as FAILED. Its timestamp + // is between the full and the successful incremental, so the cascade logic + // would find the successful incremental as "affected" if not guarded. + BackupInfo fullInfo = getBackupInfo(fullBackupId); + BackupInfo successInfo = getBackupInfo(successIncrBackupId); + long failedTs = (fullInfo.getStartTs() + successInfo.getStartTs()) / 2; + + String failedBackupId = "backup_" + failedTs; + BackupInfo failedInfo = new BackupInfo(failedBackupId, BackupType.INCREMENTAL, + tableList.toArray(new TableName[0]), BACKUP_ROOT_DIR); + failedInfo.setStartTs(failedTs); + failedInfo.setCompleteTs(failedTs + 1); + failedInfo.setState(BackupState.FAILED); + failedInfo.setFailedMsg("Simulated failure for test"); + + try (BackupSystemTable sysTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + sysTable.updateBackupInfo(failedInfo); + } + assertTrue("FAILED backup should exist", checkFailed(failedBackupId)); + LOG.info("Inserted FAILED backup record: {}", failedBackupId); + + // Step 5: Delete the FAILED backup + int deleted = getBackupAdmin().deleteBackups(new String[] { failedBackupId }); + assertEquals(1, deleted); + LOG.info("Deleted FAILED backup {}", failedBackupId); + + // Step 6: Verify the FAILED backup is gone + assertNull("FAILED backup should be deleted", getBackupInfo(failedBackupId)); + + // Step 7: Verify the successful incremental is still present + successInfo = getBackupInfo(successIncrBackupId); + assertNotNull("COMPLETE incremental backup should NOT be cascade-deleted", successInfo); + assertEquals(BackupState.COMPLETE, successInfo.getState()); + + // Step 8: Verify the full backup is still present + fullInfo = getBackupInfo(fullBackupId); + assertNotNull("Full backup should still exist", fullInfo); + assertEquals(BackupState.COMPLETE, fullInfo.getState()); + + LOG.info("Test passed: deleting FAILED backup did not cascade to COMPLETE incremental"); + } + + private BackupInfo getBackupInfo(String backupId) throws Exception { + try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { + return table.readBackupInfo(backupId); + } + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java index ba09817fcdec..ca0c0338ba75 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java @@ -17,32 +17,26 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashSet; import java.util.List; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.backup.impl.BackupManifest; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestFullBackup extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFullBackup.class); - private static final Logger LOG = LoggerFactory.getLogger(TestFullBackup.class); @Test diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java index af6ce077e051..6a598f40d7a9 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java @@ -17,30 +17,24 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestFullBackupSet extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFullBackupSet.class); - private static final Logger LOG = LoggerFactory.getLogger(TestFullBackupSet.class); /** diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java index 98e05cc5a128..b248bd8c1232 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java @@ -17,30 +17,24 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestFullBackupSetRestoreSet extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFullBackupSetRestoreSet.class); - private static final Logger LOG = LoggerFactory.getLogger(TestFullBackupSetRestoreSet.class); @Test @@ -55,15 +49,15 @@ public void testFullRestoreSetToOtherTable() throws Exception { List names = table.describeBackupSet(name); assertNotNull(names); - assertTrue(names.size() == 1); + assertEquals(1, names.size()); assertTrue(names.get(0).equals(table1)); String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-s", name }; // Run backup int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); + assertEquals(0, ret); List backups = table.getBackupHistory(); - assertTrue(backups.size() == 1); + assertEquals(1, backups.size()); String backupId = backups.get(0).getBackupId(); assertTrue(checkSucceeded(backupId)); @@ -74,7 +68,7 @@ public void testFullRestoreSetToOtherTable() throws Exception { table1_restore.getNameAsString(), "-o" }; // Run backup ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret == 0); + assertEquals(0, ret); Admin hba = TEST_UTIL.getAdmin(); assertTrue(hba.tableExists(table1_restore)); // Verify number of rows in both tables @@ -97,13 +91,13 @@ public void testFullRestoreSetToSameTable() throws Exception { List names = table.describeBackupSet(name); assertNotNull(names); - assertTrue(names.size() == 1); + assertEquals(1, names.size()); assertTrue(names.get(0).equals(table1)); String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-s", name }; // Run backup int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); + assertEquals(0, ret); List backups = table.getBackupHistory(); String backupId = backups.get(0).getBackupId(); assertTrue(checkSucceeded(backupId)); @@ -116,7 +110,7 @@ public void testFullRestoreSetToSameTable() throws Exception { args = new String[] { BACKUP_ROOT_DIR, backupId, "-s", name, "-o" }; // Run backup ret = ToolRunner.run(conf1, new RestoreDriver(), args); - assertTrue(ret == 0); + assertEquals(0, ret); Admin hba = TEST_UTIL.getAdmin(); assertTrue(hba.tableExists(table1)); // Verify number of rows in both tables diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupWithFailures.java index 1536fd1841fb..6ffd09644c26 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupWithFailures.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupWithFailures.java @@ -17,31 +17,25 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import java.util.Set; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.backup.impl.TableBackupClient; import org.apache.hadoop.hbase.backup.impl.TableBackupClient.Stage; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestFullBackupWithFailures extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFullBackupWithFailures.class); - private static final Logger LOG = LoggerFactory.getLogger(TestFullBackupWithFailures.class); @Test @@ -75,7 +69,7 @@ public void runBackupAndFailAtStage(int stage) throws Exception { assertFalse(checkSucceeded(backupId)); } Set tables = table.getIncrementalBackupTableSet(BACKUP_ROOT_DIR); - assertTrue(tables.size() == 0); + assertTrue(tables.isEmpty()); } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java index d16d7af75014..1eb26a9b82aa 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java @@ -17,34 +17,29 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.List; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestFullRestore extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFullRestore.class); - private static final Logger LOG = LoggerFactory.getLogger(TestFullRestore.class); /** @@ -293,20 +288,22 @@ public void testFullRestoreMultipleOverwriteCommand() throws Exception { * Verify that restore fails on a single table that does not exist. * @throws Exception if doing the backup or restoring it fails */ - @Test(expected = IOException.class) + @Test public void testFullRestoreSingleDNE() throws Exception { - LOG.info("test restore fails on a single table that does not exist"); - List tables = Lists.newArrayList(table1); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - LOG.info("backup complete"); - - TableName[] tableset = new TableName[] { TableName.valueOf("faketable") }; - TableName[] tablemap = new TableName[] { table1_restore }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tableset, - tablemap, false)); + assertThrows(IOException.class, () -> { + LOG.info("test restore fails on a single table that does not exist"); + List tables = Lists.newArrayList(table1); + String backupId = fullTableBackup(tables); + assertTrue(checkSucceeded(backupId)); + + LOG.info("backup complete"); + + TableName[] tableset = new TableName[] { TableName.valueOf("faketable") }; + TableName[] tablemap = new TableName[] { table1_restore }; + BackupAdmin client = getBackupAdmin(); + client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tableset, + tablemap, false)); + }); } /** @@ -335,20 +332,22 @@ public void testFullRestoreSingleDNECommand() throws Exception { * Verify that restore fails on multiple tables that do not exist. * @throws Exception if doing the backup or restoring it fails */ - @Test(expected = IOException.class) + @Test public void testFullRestoreMultipleDNE() throws Exception { - LOG.info("test restore fails on multiple tables that do not exist"); - - List tables = Lists.newArrayList(table2, table3); - String backupId = fullTableBackup(tables); - assertTrue(checkSucceeded(backupId)); - - TableName[] restore_tableset = - new TableName[] { TableName.valueOf("faketable1"), TableName.valueOf("faketable2") }; - TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - BackupAdmin client = getBackupAdmin(); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, - restore_tableset, tablemap, false)); + assertThrows(IOException.class, () -> { + LOG.info("test restore fails on multiple tables that do not exist"); + + List tables = Lists.newArrayList(table2, table3); + String backupId = fullTableBackup(tables); + assertTrue(checkSucceeded(backupId)); + + TableName[] restore_tableset = + new TableName[] { TableName.valueOf("faketable1"), TableName.valueOf("faketable2") }; + TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; + BackupAdmin client = getBackupAdmin(); + client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, + restore_tableset, tablemap, false)); + }); } /** diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestHBackupFileSystem.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestHBackupFileSystem.java index 3bdc2f620794..4939f12eb55d 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestHBackupFileSystem.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestHBackupFileSystem.java @@ -17,24 +17,18 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestHBackupFileSystem { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHBackupFileSystem.class); - private static final Path ROOT_DIR = new Path("/backup/root"); private static final String BACKUP_ID = "123"; diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java deleted file mode 100644 index df187f752959..000000000000 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java +++ /dev/null @@ -1,577 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; -import org.apache.hadoop.hbase.backup.impl.BackupManifest; -import org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.LogRoller; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.tool.BulkLoadHFiles; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.CommonFSUtils; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.HFileArchiveUtil; -import org.apache.hadoop.hbase.util.HFileTestUtil; -import org.junit.After; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hbase.thirdparty.com.google.common.base.Throwables; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -import org.apache.hbase.thirdparty.com.google.common.collect.Sets; - -@Category(LargeTests.class) -@RunWith(Parameterized.class) -public class TestIncrementalBackup extends TestBackupBase { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIncrementalBackup.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestIncrementalBackup.class); - private static final byte[] BULKLOAD_START_KEY = new byte[] { 0x00 }; - private static final byte[] BULKLOAD_END_KEY = new byte[] { Byte.MAX_VALUE }; - - @Parameterized.Parameters - public static Collection data() { - provider = "multiwal"; - List params = new ArrayList<>(); - params.add(new Object[] { Boolean.TRUE }); - return params; - } - - public TestIncrementalBackup(Boolean b) { - } - - @After - public void ensurePreviousBackupTestsAreCleanedUp() throws Exception { - TEST_UTIL.flush(table1); - TEST_UTIL.flush(table2); - - TEST_UTIL.truncateTable(table1).close(); - TEST_UTIL.truncateTable(table2).close(); - - if (TEST_UTIL.getAdmin().tableExists(table1_restore)) { - TEST_UTIL.flush(table1_restore); - TEST_UTIL.truncateTable(table1_restore).close(); - } - - TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().forEach(rst -> { - try { - LogRoller walRoller = rst.getRegionServer().getWalRoller(); - walRoller.requestRollAll(); - walRoller.waitUntilWalRollFinished(); - } catch (Exception ignored) { - } - }); - - try (Table table = TEST_UTIL.getConnection().getTable(table1)) { - loadTable(table); - } - - try (Table table = TEST_UTIL.getConnection().getTable(table2)) { - loadTable(table); - } - } - - // implement all test cases in 1 test since incremental - // backup/restore has dependencies - @Test - public void TestIncBackupRestore() throws Exception { - int ADD_ROWS = 99; - - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - List tables = Lists.newArrayList(table1, table2); - final byte[] fam3Name = Bytes.toBytes("f3"); - final byte[] mobName = Bytes.toBytes("mob"); - - TableDescriptor newTable1Desc = TableDescriptorBuilder.newBuilder(table1Desc) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam3Name)) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(mobName).setMobEnabled(true) - .setMobThreshold(5L).build()) - .build(); - TEST_UTIL.getAdmin().modifyTable(newTable1Desc); - - try (Connection conn = ConnectionFactory.createConnection(conf1)) { - int NB_ROWS_FAM3 = 6; - insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close(); - insertIntoTable(conn, table1, mobName, 3, NB_ROWS_FAM3).close(); - Admin admin = conn.getAdmin(); - BackupAdminImpl client = new BackupAdminImpl(conn); - BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = takeFullBackup(tables, client); - validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdFull); - assertTrue(checkSucceeded(backupIdFull)); - - // #2 - insert some data to table - Table t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS); - LOG.debug("writing " + ADD_ROWS + " rows to " + table1); - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3); - LOG.debug("written " + ADD_ROWS + " rows to " + table1); - // additionally, insert rows to MOB cf - int NB_ROWS_MOB = 111; - insertIntoTable(conn, table1, mobName, 3, NB_ROWS_MOB); - LOG.debug("written " + NB_ROWS_MOB + " rows to " + table1 + " to Mob enabled CF"); - t1.close(); - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_MOB); - Table t2 = conn.getTable(table2); - Put p2; - for (int i = 0; i < 5; i++) { - p2 = new Put(Bytes.toBytes("row-t2" + i)); - p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t2.put(p2); - } - Assert.assertEquals(NB_ROWS_IN_BATCH + 5, TEST_UTIL.countRows(t2)); - t2.close(); - LOG.debug("written " + 5 + " rows to " + table2); - // split table1 - MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); - List regions = cluster.getRegions(table1); - byte[] name = regions.get(0).getRegionInfo().getRegionName(); - long startSplitTime = EnvironmentEdgeManager.currentTime(); - try { - admin.splitRegionAsync(name).get(); - } catch (Exception e) { - // although split fail, this may not affect following check in current API, - // exception will be thrown. - LOG.debug("region is not splittable, because " + e); - } - TEST_UTIL.waitTableAvailable(table1); - long endSplitTime = EnvironmentEdgeManager.currentTime(); - // split finished - LOG.debug("split finished in =" + (endSplitTime - startSplitTime)); - - // #3 - incremental backup for multiple tables - tables = Lists.newArrayList(table1, table2); - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); - assertTrue(checkSucceeded(backupIdIncMultiple)); - BackupManifest manifest = - HBackupFileSystem.getManifest(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple); - assertEquals(Sets.newHashSet(table1, table2), new HashSet<>(manifest.getTableList())); - validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdIncMultiple); - - // add column family f2 to table1 - // drop column family f3 - final byte[] fam2Name = Bytes.toBytes("f2"); - newTable1Desc = TableDescriptorBuilder.newBuilder(newTable1Desc) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2Name)).removeColumnFamily(fam3Name) - .build(); - TEST_UTIL.getAdmin().modifyTable(newTable1Desc); - - // check that an incremental backup fails because the CFs don't match - final List tablesCopy = tables; - IOException ex = assertThrows(IOException.class, () -> client - .backupTables(createBackupRequest(BackupType.INCREMENTAL, tablesCopy, BACKUP_ROOT_DIR))); - checkThrowsCFMismatch(ex, ImmutableList.of(table1)); - takeFullBackup(tables, client); - - int NB_ROWS_FAM2 = 7; - Table t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2); - t3.close(); - - // Wait for 5 sec to make sure that old WALs were deleted - Thread.sleep(5000); - - // #4 - additional incremental backup for multiple tables - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple2 = client.backupTables(request); - assertTrue(checkSucceeded(backupIdIncMultiple2)); - validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdIncMultiple2); - - // #5 - restore full backup for all tables - TableName[] tablesRestoreFull = new TableName[] { table1, table2 }; - TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore }; - - LOG.debug("Restoring full " + backupIdFull); - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false, - tablesRestoreFull, tablesMapFull, true)); - - // #6.1 - check tables for full restore - Admin hAdmin = TEST_UTIL.getAdmin(); - assertTrue(hAdmin.tableExists(table1_restore)); - assertTrue(hAdmin.tableExists(table2_restore)); - hAdmin.close(); - - // #6.2 - checking row count of tables for full restore - Table hTable = conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + NB_ROWS_FAM3); - hTable.close(); - - hTable = conn.getTable(table2_restore); - Assert.assertEquals(NB_ROWS_IN_BATCH, TEST_UTIL.countRows(hTable)); - hTable.close(); - - // #7 - restore incremental backup for multiple tables, with overwrite - TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 }; - TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore }; - client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2, false, - tablesRestoreIncMultiple, tablesMapIncMultiple, true)); - hTable = conn.getTable(table1_restore); - - LOG.debug("After incremental restore: " + hTable.getDescriptor()); - int countFamName = TEST_UTIL.countRows(hTable, famName); - LOG.debug("f1 has " + countFamName + " rows"); - Assert.assertEquals(countFamName, NB_ROWS_IN_BATCH + ADD_ROWS); - - int countFam2Name = TEST_UTIL.countRows(hTable, fam2Name); - LOG.debug("f2 has " + countFam2Name + " rows"); - Assert.assertEquals(countFam2Name, NB_ROWS_FAM2); - - int countMobName = TEST_UTIL.countRows(hTable, mobName); - LOG.debug("mob has " + countMobName + " rows"); - Assert.assertEquals(countMobName, NB_ROWS_MOB); - hTable.close(); - - hTable = conn.getTable(table2_restore); - Assert.assertEquals(NB_ROWS_IN_BATCH + 5, TEST_UTIL.countRows(hTable)); - hTable.close(); - admin.close(); - } - } - - @Test - public void TestIncBackupRestoreWithOriginalSplits() throws Exception { - byte[] mobFam = Bytes.toBytes("mob"); - - List tables = Lists.newArrayList(table1); - TableDescriptor newTable1Desc = - TableDescriptorBuilder.newBuilder(table1Desc).setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(mobFam).setMobEnabled(true).setMobThreshold(5L).build()).build(); - TEST_UTIL.getAdmin().modifyTable(newTable1Desc); - - Connection conn = TEST_UTIL.getConnection(); - BackupAdminImpl backupAdmin = new BackupAdminImpl(conn); - BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String fullBackupId = backupAdmin.backupTables(request); - assertTrue(checkSucceeded(fullBackupId)); - - TableName[] fromTables = new TableName[] { table1 }; - TableName[] toTables = new TableName[] { table1_restore }; - - List preRestoreBackupFiles = getBackupFiles(); - backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, fullBackupId, false, - fromTables, toTables, true, true)); - List postRestoreBackupFiles = getBackupFiles(); - - // Check that the backup files are the same before and after the restore process - Assert.assertEquals(postRestoreBackupFiles, preRestoreBackupFiles); - Assert.assertEquals(TEST_UTIL.countRows(table1_restore), NB_ROWS_IN_BATCH); - - int ROWS_TO_ADD = 1_000; - // different IDs so that rows don't overlap - insertIntoTable(conn, table1, famName, 3, ROWS_TO_ADD); - insertIntoTable(conn, table1, mobFam, 4, ROWS_TO_ADD); - - try (Admin admin = conn.getAdmin()) { - List currentRegions = TEST_UTIL.getHBaseCluster().getRegions(table1); - for (HRegion region : currentRegions) { - byte[] name = region.getRegionInfo().getEncodedNameAsBytes(); - admin.splitRegionAsync(name).get(); - } - - TEST_UTIL.waitTableAvailable(table1); - - // Make sure we've split regions - assertNotEquals(currentRegions, TEST_UTIL.getHBaseCluster().getRegions(table1)); - - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String incrementalBackupId = backupAdmin.backupTables(request); - assertTrue(checkSucceeded(incrementalBackupId)); - preRestoreBackupFiles = getBackupFiles(); - backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, - false, fromTables, toTables, true, true)); - postRestoreBackupFiles = getBackupFiles(); - Assert.assertEquals(postRestoreBackupFiles, preRestoreBackupFiles); - Assert.assertEquals(NB_ROWS_IN_BATCH + ROWS_TO_ADD + ROWS_TO_ADD, - TEST_UTIL.countRows(table1_restore)); - - // test bulkloads - HRegion regionToBulkload = TEST_UTIL.getHBaseCluster().getRegions(table1).get(0); - String regionName = regionToBulkload.getRegionInfo().getEncodedName(); - - insertIntoTable(conn, table1, famName, 5, ROWS_TO_ADD); - insertIntoTable(conn, table1, mobFam, 6, ROWS_TO_ADD); - - doBulkload(table1, regionName, famName, mobFam); - - // we need to major compact the regions to make sure there are no references - // and the regions are once again splittable - TEST_UTIL.compact(true); - TEST_UTIL.flush(); - TEST_UTIL.waitTableAvailable(table1); - - for (HRegion region : TEST_UTIL.getHBaseCluster().getRegions(table1)) { - if (region.isSplittable()) { - admin.splitRegionAsync(region.getRegionInfo().getEncodedNameAsBytes()).get(); - } - } - - request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - incrementalBackupId = backupAdmin.backupTables(request); - assertTrue(checkSucceeded(incrementalBackupId)); - - preRestoreBackupFiles = getBackupFiles(); - backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, - false, fromTables, toTables, true, true)); - postRestoreBackupFiles = getBackupFiles(); - - Assert.assertEquals(postRestoreBackupFiles, preRestoreBackupFiles); - - int rowsExpected = TEST_UTIL.countRows(table1); - int rowsActual = TEST_UTIL.countRows(table1_restore); - - Assert.assertEquals(rowsExpected, rowsActual); - } - } - - @Test - public void TestIncBackupRestoreWithOriginalSplitsSeperateFs() throws Exception { - String originalBackupRoot = BACKUP_ROOT_DIR; - // prepare BACKUP_ROOT_DIR on a different filesystem from HBase. - try (Connection conn = ConnectionFactory.createConnection(conf1); - BackupAdminImpl admin = new BackupAdminImpl(conn)) { - String backupTargetDir = TEST_UTIL.getDataTestDir("backupTarget").toString(); - BACKUP_ROOT_DIR = new File(backupTargetDir).toURI().toString(); - - List tables = Lists.newArrayList(table1); - - insertIntoTable(conn, table1, famName, 3, 100); - String fullBackupId = takeFullBackup(tables, admin, true); - assertTrue(checkSucceeded(fullBackupId)); - - insertIntoTable(conn, table1, famName, 4, 100); - - HRegion regionToBulkload = TEST_UTIL.getHBaseCluster().getRegions(table1).get(0); - String regionName = regionToBulkload.getRegionInfo().getEncodedName(); - doBulkload(table1, regionName, famName); - - BackupRequest request = - createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true); - String incrementalBackupId = admin.backupTables(request); - assertTrue(checkSucceeded(incrementalBackupId)); - - TableName[] fromTable = new TableName[] { table1 }; - TableName[] toTable = new TableName[] { table1_restore }; - - // Using original splits - admin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, false, - fromTable, toTable, true, true)); - - int actualRowCount = TEST_UTIL.countRows(table1_restore); - int expectedRowCount = TEST_UTIL.countRows(table1); - assertEquals(expectedRowCount, actualRowCount); - - // Using new splits - admin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, false, - fromTable, toTable, true, false)); - - expectedRowCount = TEST_UTIL.countRows(table1); - assertEquals(expectedRowCount, actualRowCount); - - } finally { - BACKUP_ROOT_DIR = originalBackupRoot; - } - - } - - @Test - public void TestIncBackupRestoreHandlesArchivedFiles() throws Exception { - byte[] fam2 = Bytes.toBytes("f2"); - TableDescriptor newTable1Desc = TableDescriptorBuilder.newBuilder(table1Desc) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).build()).build(); - TEST_UTIL.getAdmin().modifyTable(newTable1Desc); - try (Connection conn = ConnectionFactory.createConnection(conf1); - BackupAdminImpl admin = new BackupAdminImpl(conn)) { - String backupTargetDir = TEST_UTIL.getDataTestDir("backupTarget").toString(); - BACKUP_ROOT_DIR = new File(backupTargetDir).toURI().toString(); - - List tables = Lists.newArrayList(table1); - - insertIntoTable(conn, table1, famName, 3, 100); - String fullBackupId = takeFullBackup(tables, admin, true); - assertTrue(checkSucceeded(fullBackupId)); - - insertIntoTable(conn, table1, famName, 4, 100); - - HRegion regionToBulkload = TEST_UTIL.getHBaseCluster().getRegions(table1).get(0); - String regionName = regionToBulkload.getRegionInfo().getEncodedName(); - // Requires a mult-fam bulkload to ensure we're appropriately handling - // multi-file bulkloads - Path regionDir = doBulkload(table1, regionName, famName, fam2); - - // archive the files in the region directory - Path archiveDir = - HFileArchiveUtil.getStoreArchivePath(conf1, table1, regionName, Bytes.toString(famName)); - TEST_UTIL.getTestFileSystem().mkdirs(archiveDir); - RemoteIterator iter = - TEST_UTIL.getTestFileSystem().listFiles(regionDir, true); - List paths = new ArrayList<>(); - while (iter.hasNext()) { - Path path = iter.next().getPath(); - if (path.toString().contains("_SeqId_")) { - paths.add(path); - } - } - assertTrue(paths.size() > 1); - Path path = paths.get(0); - String name = path.toString(); - int startIdx = name.lastIndexOf(Path.SEPARATOR); - String filename = name.substring(startIdx + 1); - Path archiveFile = new Path(archiveDir, filename); - // archive 1 of the files - boolean success = TEST_UTIL.getTestFileSystem().rename(path, archiveFile); - assertTrue(success); - assertTrue(TEST_UTIL.getTestFileSystem().exists(archiveFile)); - assertFalse(TEST_UTIL.getTestFileSystem().exists(path)); - - BackupRequest request = - createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true); - String incrementalBackupId = admin.backupTables(request); - assertTrue(checkSucceeded(incrementalBackupId)); - - TableName[] fromTable = new TableName[] { table1 }; - TableName[] toTable = new TableName[] { table1_restore }; - - admin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, false, - fromTable, toTable, true)); - - int actualRowCount = TEST_UTIL.countRows(table1_restore); - int expectedRowCount = TEST_UTIL.countRows(table1); - assertEquals(expectedRowCount, actualRowCount); - } - } - - private void checkThrowsCFMismatch(IOException ex, List tables) { - Throwable cause = Throwables.getRootCause(ex); - assertEquals(cause.getClass(), ColumnFamilyMismatchException.class); - ColumnFamilyMismatchException e = (ColumnFamilyMismatchException) cause; - assertEquals(tables, e.getMismatchedTables()); - } - - private String takeFullBackup(List tables, BackupAdminImpl backupAdmin) - throws IOException { - return takeFullBackup(tables, backupAdmin, false); - } - - private String takeFullBackup(List tables, BackupAdminImpl backupAdmin, - boolean noChecksumVerify) throws IOException { - BackupRequest req = - createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR, noChecksumVerify); - String backupId = backupAdmin.backupTables(req); - checkSucceeded(backupId); - return backupId; - } - - private static Path doBulkload(TableName tn, String regionName, byte[]... fams) - throws IOException { - Path regionDir = createHFiles(tn, regionName, fams); - Map results = - BulkLoadHFiles.create(conf1).bulkLoad(tn, regionDir); - assertFalse(results.isEmpty()); - return regionDir; - } - - private static Path createHFiles(TableName tn, String regionName, byte[]... fams) - throws IOException { - Path rootdir = CommonFSUtils.getRootDir(conf1); - Path regionDir = CommonFSUtils.getRegionDir(rootdir, tn, regionName); - - FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); - fs.mkdirs(rootdir); - - for (byte[] fam : fams) { - Path famDir = new Path(regionDir, Bytes.toString(fam)); - Path hFileDir = new Path(famDir, UUID.randomUUID().toString()); - HFileTestUtil.createHFile(conf1, fs, hFileDir, fam, qualName, BULKLOAD_START_KEY, - BULKLOAD_END_KEY, 1000); - } - - return regionDir; - } - - /** - * Check that backup manifest can be produced for a different root. Users may want to move - * existing backups to a different location. - */ - private void validateRootPathCanBeOverridden(String originalPath, String backupId) - throws IOException { - String anotherRootDir = "/some/other/root/dir"; - Path anotherPath = new Path(anotherRootDir, backupId); - BackupManifest.BackupImage differentLocationImage = BackupManifest.hydrateRootDir( - HBackupFileSystem.getManifest(conf1, new Path(originalPath), backupId).getBackupImage(), - anotherPath); - assertEquals(differentLocationImage.getRootDir(), anotherRootDir); - for (BackupManifest.BackupImage ancestor : differentLocationImage.getAncestors()) { - assertEquals(anotherRootDir, ancestor.getRootDir()); - } - } - - private List getBackupFiles() throws IOException { - FileSystem fs = TEST_UTIL.getTestFileSystem(); - RemoteIterator iter = fs.listFiles(new Path(BACKUP_ROOT_DIR), true); - List files = new ArrayList<>(); - - while (iter.hasNext()) { - files.add(iter.next()); - } - - return files; - } -} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java index a5eec87fb06b..5bd69f57a821 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; import org.apache.hadoop.hbase.backup.util.BackupUtils; @@ -31,10 +31,8 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,13 +42,9 @@ * 1. Create table t1, t2 2. Load data to t1, t2 3 Full backup t1, t2 4 Delete t2 5 Load data to t1 * 6 Incremental backup t1 */ -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestIncrementalBackupDeleteTable extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIncrementalBackupDeleteTable.class); - private static final Logger LOG = LoggerFactory.getLogger(TestIncrementalBackupDeleteTable.class); // implement all test cases in 1 test since incremental backup/restore has dependencies @@ -78,7 +72,7 @@ public void testIncBackupDeleteTable() throws Exception { t1.put(p1); } - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH * 2); + assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH * 2); t1.close(); // Delete table table2 @@ -106,11 +100,11 @@ public void testIncBackupDeleteTable() throws Exception { // #5.2 - checking row count of tables for full restore Table hTable = conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH); + assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH); hTable.close(); hTable = conn.getTable(table2_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH); + assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH); hTable.close(); // #6 - restore incremental backup for table1 @@ -120,7 +114,7 @@ public void testIncBackupDeleteTable() throws Exception { tablesRestoreIncMultiple, tablesMapIncMultiple, true)); hTable = conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH * 2); + assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH * 2); hTable.close(); admin.close(); conn.close(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithBulkLoad.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithBulkLoad.java index c383a8545a8b..4d4076f28c05 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithBulkLoad.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithBulkLoad.java @@ -34,13 +34,13 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; @@ -55,40 +55,36 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testing.TestingHBaseCluster; import org.apache.hadoop.hbase.testing.TestingHBaseClusterOption; import org.apache.hadoop.hbase.tool.BulkLoadHFiles; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.params.provider.Arguments; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(MediumTests.class) -@RunWith(Parameterized.class) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: useBulkLoad={0}") public class TestIncrementalBackupMergeWithBulkLoad { private static final Logger LOG = LoggerFactory.getLogger(TestIncrementalBackupMergeWithBulkLoad.class); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIncrementalBackupMergeWithBulkLoad.class); + public boolean useBulkLoad; - @Parameterized.Parameters(name = "{index}: useBulkLoad={0}") - public static Iterable data() { - return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; + public TestIncrementalBackupMergeWithBulkLoad(boolean useBulkLoad) { + this.useBulkLoad = useBulkLoad; } - @Parameterized.Parameter(0) - public boolean useBulkLoad; + public static Stream parameters() { + return Stream.of(Arguments.of(true), Arguments.of(false)); + } private TableName sourceTable; private TableName targetTable; @@ -98,7 +94,7 @@ public static Iterable data() { private static final Path BACKUP_ROOT_DIR = new Path("backupIT"); private static final byte[] COLUMN_FAMILY = Bytes.toBytes("0"); - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { Configuration conf = HBaseConfiguration.create(); enableBackup(conf); @@ -106,12 +102,12 @@ public static void beforeClass() throws Exception { cluster.start(); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { cluster.stop(); } - @Before + @BeforeEach public void setUp() throws Exception { sourceTable = TableName.valueOf("table-" + useBulkLoad); targetTable = TableName.valueOf("another-table-" + useBulkLoad); @@ -120,7 +116,7 @@ public void setUp() throws Exception { createTable(targetTable); } - @Test + @TestTemplate public void testMergeContainingBulkloadedHfiles() throws Exception { Instant timestamp = Instant.now(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java index 1ece1770489b..ca066d436c2f 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hbase.backup; import static org.apache.hadoop.hbase.backup.util.BackupUtils.succeeded; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; import org.apache.hadoop.hbase.backup.impl.BackupCommands; @@ -42,22 +41,17 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Pair; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestIncrementalBackupMergeWithFailures extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIncrementalBackupMergeWithFailures.class); - private static final Logger LOG = LoggerFactory.getLogger(TestIncrementalBackupMergeWithFailures.class); @@ -80,7 +74,7 @@ public void setConf(Configuration conf) { if (val != null) { failurePhase = FailurePhase.valueOf(val); } else { - Assert.fail("Failure phase is not set"); + Assertions.fail("Failure phase is not set"); } } @@ -248,13 +242,13 @@ public void TestIncBackupMergeRestore() throws Exception { Table t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS); LOG.debug("writing " + ADD_ROWS + " rows to " + table1); - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS); + Assertions.assertEquals(NB_ROWS_IN_BATCH + ADD_ROWS, TEST_UTIL.countRows(t1)); t1.close(); LOG.debug("written " + ADD_ROWS + " rows to " + table1); Table t2 = insertIntoTable(conn, table2, famName, 1, ADD_ROWS); - Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + ADD_ROWS); + Assertions.assertEquals(NB_ROWS_IN_BATCH + ADD_ROWS, TEST_UTIL.countRows(t2)); t2.close(); LOG.debug("written " + ADD_ROWS + " rows to " + table2); @@ -285,7 +279,7 @@ public void TestIncBackupMergeRestore() throws Exception { try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) { String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 }; bAdmin.mergeBackups(backups); - Assert.fail("Expected IOException"); + Assertions.fail("Expected IOException"); } catch (IOException e) { BackupSystemTable table = new BackupSystemTable(conn); if (phase.ordinal() < FailurePhase.PHASE4.ordinal()) { @@ -294,7 +288,7 @@ public void TestIncBackupMergeRestore() throws Exception { assertFalse(table.isMergeInProgress()); try { table.finishBackupExclusiveOperation(); - Assert.fail("IOException is expected"); + Assertions.fail("IOException is expected"); } catch (IOException ee) { // Expected } @@ -303,7 +297,7 @@ public void TestIncBackupMergeRestore() throws Exception { assertTrue(table.isMergeInProgress()); try { table.startBackupExclusiveOperation(); - Assert.fail("IOException is expected"); + Assertions.fail("IOException is expected"); } catch (IOException ee) { // Expected - clean up before proceeding // table.finishMergeOperation(); @@ -336,12 +330,12 @@ public void TestIncBackupMergeRestore() throws Exception { Table hTable = conn.getTable(table1_restore); LOG.debug("After incremental restore: " + hTable.getDescriptor()); LOG.debug("f1 has " + TEST_UTIL.countRows(hTable, famName) + " rows"); - Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH + 2 * ADD_ROWS); + Assertions.assertEquals(NB_ROWS_IN_BATCH + 2 * ADD_ROWS, TEST_UTIL.countRows(hTable, famName)); hTable.close(); hTable = conn.getTable(table2_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 2 * ADD_ROWS); + Assertions.assertEquals(NB_ROWS_IN_BATCH + 2 * ADD_ROWS, TEST_UTIL.countRows(hTable)); hTable.close(); admin.close(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestore.java new file mode 100644 index 000000000000..343c371e6b07 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestore.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.impl.BackupManifest; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; + +@Tag(LargeTests.TAG) +public class TestIncrementalBackupRestore extends IncrementalBackupRestoreTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(TestIncrementalBackupRestore.class); + + // implement all test cases in 1 test since incremental + // backup/restore has dependencies + @Test + public void testIncBackupRestore() throws Exception { + int ADD_ROWS = 99; + + // #1 - create full backup for all tables + LOG.info("create full backup image for all tables"); + List tables = Lists.newArrayList(table1, table2); + final byte[] fam3Name = Bytes.toBytes("f3"); + final byte[] mobName = Bytes.toBytes("mob"); + + TableDescriptor newTable1Desc = TableDescriptorBuilder.newBuilder(table1Desc) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam3Name)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(mobName).setMobEnabled(true) + .setMobThreshold(5L).build()) + .build(); + TEST_UTIL.getAdmin().modifyTable(newTable1Desc); + + try (Connection conn = ConnectionFactory.createConnection(conf1); + Admin admin = conn.getAdmin()) { + int NB_ROWS_FAM3 = 6; + insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close(); + insertIntoTable(conn, table1, mobName, 3, NB_ROWS_FAM3).close(); + + BackupAdminImpl client = new BackupAdminImpl(conn); + BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); + String backupIdFull = takeFullBackup(tables, client); + validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdFull); + assertTrue(checkSucceeded(backupIdFull)); + + // #2 - insert some data to table + Table t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS); + LOG.debug("writing {} rows to {}", ADD_ROWS, table1); + assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3); + LOG.debug("written {} rows to {}", ADD_ROWS, table1); + // additionally, insert rows to MOB cf + int NB_ROWS_MOB = 111; + insertIntoTable(conn, table1, mobName, 3, NB_ROWS_MOB); + LOG.debug("written {} rows to {} to Mob enabled CF", NB_ROWS_MOB, table1); + t1.close(); + assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_MOB); + Table t2 = conn.getTable(table2); + Put p2; + for (int i = 0; i < 5; i++) { + p2 = new Put(Bytes.toBytes("row-t2" + i)); + p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t2.put(p2); + } + assertEquals(NB_ROWS_IN_BATCH + 5, TEST_UTIL.countRows(t2)); + t2.close(); + LOG.debug("written 5 rows to {}", table2); + // split table1 + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + List regions = cluster.getRegions(table1); + byte[] name = regions.get(0).getRegionInfo().getRegionName(); + long startSplitTime = EnvironmentEdgeManager.currentTime(); + try { + admin.splitRegionAsync(name).get(); + } catch (Exception e) { + // although split fail, this may not affect following check in current API, + // exception will be thrown. + LOG.debug("region is not splittable, because " + e); + } + TEST_UTIL.waitTableAvailable(table1); + long endSplitTime = EnvironmentEdgeManager.currentTime(); + // split finished + LOG.debug("split finished in = {}", endSplitTime - startSplitTime); + + // #3 - incremental backup for multiple tables + tables = Lists.newArrayList(table1, table2); + request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); + String backupIdIncMultiple = client.backupTables(request); + assertTrue(checkSucceeded(backupIdIncMultiple)); + BackupManifest manifest = + HBackupFileSystem.getManifest(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple); + assertEquals(Sets.newHashSet(table1, table2), new HashSet<>(manifest.getTableList())); + validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdIncMultiple); + + // add column family f2 to table1 + // drop column family f3 + final byte[] fam2Name = Bytes.toBytes("f2"); + newTable1Desc = TableDescriptorBuilder.newBuilder(newTable1Desc) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2Name)).removeColumnFamily(fam3Name) + .build(); + TEST_UTIL.getAdmin().modifyTable(newTable1Desc); + + // check that an incremental backup fails because the CFs don't match + final List tablesCopy = tables; + IOException ex = assertThrows(IOException.class, () -> client + .backupTables(createBackupRequest(BackupType.INCREMENTAL, tablesCopy, BACKUP_ROOT_DIR))); + checkThrowsCFMismatch(ex, ImmutableList.of(table1)); + takeFullBackup(tables, client); + + int NB_ROWS_FAM2 = 7; + Table t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2); + t3.close(); + + // Wait for 5 sec to make sure that old WALs were deleted + Thread.sleep(5000); + + // #4 - additional incremental backup for multiple tables + request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); + String backupIdIncMultiple2 = client.backupTables(request); + assertTrue(checkSucceeded(backupIdIncMultiple2)); + validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdIncMultiple2); + + // #5 - restore full backup for all tables + TableName[] tablesRestoreFull = new TableName[] { table1, table2 }; + TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore }; + + LOG.debug("Restoring full {}", backupIdFull); + client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false, + tablesRestoreFull, tablesMapFull, true)); + + // #6.1 - check tables for full restore + Admin hAdmin = TEST_UTIL.getAdmin(); + assertTrue(hAdmin.tableExists(table1_restore)); + assertTrue(hAdmin.tableExists(table2_restore)); + + // #6.2 - checking row count of tables for full restore + try (Table hTable = conn.getTable(table1_restore)) { + assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + NB_ROWS_FAM3); + } + + try (Table hTable = conn.getTable(table2_restore)) { + assertEquals(NB_ROWS_IN_BATCH, TEST_UTIL.countRows(hTable)); + } + + // #7 - restore incremental backup for multiple tables, with overwrite + TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 }; + TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore }; + client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2, false, + tablesRestoreIncMultiple, tablesMapIncMultiple, true)); + try (Table hTable = conn.getTable(table1_restore)) { + LOG.debug("After incremental restore: {}", hTable.getDescriptor()); + int countFamName = TEST_UTIL.countRows(hTable, famName); + LOG.debug("f1 has " + countFamName + " rows"); + assertEquals(NB_ROWS_IN_BATCH + ADD_ROWS, countFamName); + + int countFam2Name = TEST_UTIL.countRows(hTable, fam2Name); + LOG.debug("f2 has {} rows", countFam2Name); + assertEquals(NB_ROWS_FAM2, countFam2Name); + + int countMobName = TEST_UTIL.countRows(hTable, mobName); + LOG.debug("mob has {} rows", countMobName); + assertEquals(NB_ROWS_MOB, countMobName); + } + + try (Table hTable = conn.getTable(table2_restore)) { + assertEquals(NB_ROWS_IN_BATCH + 5, TEST_UTIL.countRows(hTable)); + } + } + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreHandlesArchivedFiles.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreHandlesArchivedFiles.java new file mode 100644 index 000000000000..8e9deed2ab83 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreHandlesArchivedFiles.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +@Tag(LargeTests.TAG) +public class TestIncrementalBackupRestoreHandlesArchivedFiles + extends IncrementalBackupRestoreTestBase { + + @Test + public void testIncBackupRestoreHandlesArchivedFiles() throws Exception { + byte[] fam2 = Bytes.toBytes("f2"); + TableDescriptor newTable1Desc = TableDescriptorBuilder.newBuilder(table1Desc) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).build()).build(); + TEST_UTIL.getAdmin().modifyTable(newTable1Desc); + try (Connection conn = ConnectionFactory.createConnection(conf1); + BackupAdminImpl admin = new BackupAdminImpl(conn)) { + String backupTargetDir = TEST_UTIL.getDataTestDir("backupTarget").toString(); + BACKUP_ROOT_DIR = new File(backupTargetDir).toURI().toString(); + + List tables = Lists.newArrayList(table1); + + insertIntoTable(conn, table1, famName, 3, 100); + String fullBackupId = takeFullBackup(tables, admin, true); + assertTrue(checkSucceeded(fullBackupId)); + + insertIntoTable(conn, table1, famName, 4, 100); + + HRegion regionToBulkload = TEST_UTIL.getHBaseCluster().getRegions(table1).get(0); + String regionName = regionToBulkload.getRegionInfo().getEncodedName(); + // Requires a mult-fam bulkload to ensure we're appropriately handling + // multi-file bulkloads + Path regionDir = doBulkload(table1, regionName, famName, fam2); + + // archive the files in the region directory + Path archiveDir = + HFileArchiveUtil.getStoreArchivePath(conf1, table1, regionName, Bytes.toString(famName)); + TEST_UTIL.getTestFileSystem().mkdirs(archiveDir); + RemoteIterator iter = + TEST_UTIL.getTestFileSystem().listFiles(regionDir, true); + List paths = new ArrayList<>(); + while (iter.hasNext()) { + Path path = iter.next().getPath(); + if (path.toString().contains("_SeqId_")) { + paths.add(path); + } + } + assertTrue(paths.size() > 1); + Path path = paths.get(0); + String name = path.toString(); + int startIdx = name.lastIndexOf(Path.SEPARATOR); + String filename = name.substring(startIdx + 1); + Path archiveFile = new Path(archiveDir, filename); + // archive 1 of the files + boolean success = TEST_UTIL.getTestFileSystem().rename(path, archiveFile); + assertTrue(success); + assertTrue(TEST_UTIL.getTestFileSystem().exists(archiveFile)); + assertFalse(TEST_UTIL.getTestFileSystem().exists(path)); + + BackupRequest request = + createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true); + String incrementalBackupId = admin.backupTables(request); + assertTrue(checkSucceeded(incrementalBackupId)); + + TableName[] fromTable = new TableName[] { table1 }; + TableName[] toTable = new TableName[] { table1_restore }; + + admin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, false, + fromTable, toTable, true)); + + int actualRowCount = TEST_UTIL.countRows(table1_restore); + int expectedRowCount = TEST_UTIL.countRows(table1); + assertEquals(expectedRowCount, actualRowCount); + } + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreWithOriginalSplits.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreWithOriginalSplits.java new file mode 100644 index 000000000000..68ea52da462e --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreWithOriginalSplits.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +@Tag(LargeTests.TAG) +public class TestIncrementalBackupRestoreWithOriginalSplits + extends IncrementalBackupRestoreTestBase { + + @Test + public void testIncBackupRestoreWithOriginalSplits() throws Exception { + byte[] mobFam = Bytes.toBytes("mob"); + + List tables = Lists.newArrayList(table1); + TableDescriptor newTable1Desc = + TableDescriptorBuilder.newBuilder(table1Desc).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(mobFam).setMobEnabled(true).setMobThreshold(5L).build()).build(); + TEST_UTIL.getAdmin().modifyTable(newTable1Desc); + + try (Connection conn = TEST_UTIL.getConnection(); + BackupAdminImpl backupAdmin = new BackupAdminImpl(conn); Admin admin = conn.getAdmin()) { + BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); + String fullBackupId = backupAdmin.backupTables(request); + assertTrue(checkSucceeded(fullBackupId)); + + TableName[] fromTables = new TableName[] { table1 }; + TableName[] toTables = new TableName[] { table1_restore }; + + List preRestoreBackupFiles = getBackupFiles(); + backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, fullBackupId, false, + fromTables, toTables, true, true)); + List postRestoreBackupFiles = getBackupFiles(); + + // Check that the backup files are the same before and after the restore process + assertEquals(postRestoreBackupFiles, preRestoreBackupFiles); + assertEquals(NB_ROWS_IN_BATCH, TEST_UTIL.countRows(table1_restore)); + + int ROWS_TO_ADD = 1_000; + // different IDs so that rows don't overlap + insertIntoTable(conn, table1, famName, 3, ROWS_TO_ADD); + insertIntoTable(conn, table1, mobFam, 4, ROWS_TO_ADD); + List currentRegions = TEST_UTIL.getHBaseCluster().getRegions(table1); + for (HRegion region : currentRegions) { + byte[] name = region.getRegionInfo().getEncodedNameAsBytes(); + admin.splitRegionAsync(name).get(); + } + + TEST_UTIL.waitTableAvailable(table1); + + // Make sure we've split regions + assertNotEquals(currentRegions, TEST_UTIL.getHBaseCluster().getRegions(table1)); + + request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); + String incrementalBackupId = backupAdmin.backupTables(request); + assertTrue(checkSucceeded(incrementalBackupId)); + preRestoreBackupFiles = getBackupFiles(); + backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, + false, fromTables, toTables, true, true)); + postRestoreBackupFiles = getBackupFiles(); + assertEquals(postRestoreBackupFiles, preRestoreBackupFiles); + assertEquals(NB_ROWS_IN_BATCH + ROWS_TO_ADD + ROWS_TO_ADD, + TEST_UTIL.countRows(table1_restore)); + + // test bulkloads + HRegion regionToBulkload = TEST_UTIL.getHBaseCluster().getRegions(table1).get(0); + String regionName = regionToBulkload.getRegionInfo().getEncodedName(); + + insertIntoTable(conn, table1, famName, 5, ROWS_TO_ADD); + insertIntoTable(conn, table1, mobFam, 6, ROWS_TO_ADD); + + doBulkload(table1, regionName, famName, mobFam); + + // we need to major compact the regions to make sure there are no references + // and the regions are once again splittable + TEST_UTIL.compact(true); + TEST_UTIL.flush(); + TEST_UTIL.waitTableAvailable(table1); + + for (HRegion region : TEST_UTIL.getHBaseCluster().getRegions(table1)) { + if (region.isSplittable()) { + admin.splitRegionAsync(region.getRegionInfo().getEncodedNameAsBytes()).get(); + } + } + + request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); + incrementalBackupId = backupAdmin.backupTables(request); + assertTrue(checkSucceeded(incrementalBackupId)); + + preRestoreBackupFiles = getBackupFiles(); + backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, + false, fromTables, toTables, true, true)); + postRestoreBackupFiles = getBackupFiles(); + + assertEquals(postRestoreBackupFiles, preRestoreBackupFiles); + + int rowsExpected = TEST_UTIL.countRows(table1); + int rowsActual = TEST_UTIL.countRows(table1_restore); + + assertEquals(rowsExpected, rowsActual); + } + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreWithOriginalSplitsSeperateFs.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreWithOriginalSplitsSeperateFs.java new file mode 100644 index 000000000000..24dd0aa4e2de --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestoreWithOriginalSplitsSeperateFs.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.util.List; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +@Tag(LargeTests.TAG) +public class TestIncrementalBackupRestoreWithOriginalSplitsSeperateFs + extends IncrementalBackupRestoreTestBase { + + @Test + public void testIncBackupRestoreWithOriginalSplitsSeperateFs() throws Exception { + // prepare BACKUP_ROOT_DIR on a different filesystem from HBase. + try (Connection conn = ConnectionFactory.createConnection(conf1); + BackupAdminImpl admin = new BackupAdminImpl(conn)) { + String backupTargetDir = TEST_UTIL.getDataTestDir("backupTarget").toString(); + BACKUP_ROOT_DIR = new File(backupTargetDir).toURI().toString(); + + List tables = Lists.newArrayList(table1); + + insertIntoTable(conn, table1, famName, 3, 100); + String fullBackupId = takeFullBackup(tables, admin, true); + assertTrue(checkSucceeded(fullBackupId)); + + insertIntoTable(conn, table1, famName, 4, 100); + + HRegion regionToBulkload = TEST_UTIL.getHBaseCluster().getRegions(table1).get(0); + String regionName = regionToBulkload.getRegionInfo().getEncodedName(); + doBulkload(table1, regionName, famName); + + BackupRequest request = + createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true); + String incrementalBackupId = admin.backupTables(request); + assertTrue(checkSucceeded(incrementalBackupId)); + + TableName[] fromTable = new TableName[] { table1 }; + TableName[] toTable = new TableName[] { table1_restore }; + + // Using original splits + admin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, false, + fromTable, toTable, true, true)); + + int actualRowCount = TEST_UTIL.countRows(table1_restore); + int expectedRowCount = TEST_UTIL.countRows(table1); + assertEquals(expectedRowCount, actualRowCount); + + // Using new splits + admin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, false, + fromTable, toTable, true, false)); + + expectedRowCount = TEST_UTIL.countRows(table1); + assertEquals(expectedRowCount, actualRowCount); + } + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java new file mode 100644 index 000000000000..d877a38f9627 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.backup.impl.BulkLoad; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.tool.BulkLoadHFiles; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.hbase.util.HFileTestUtil; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + +/** + * This test checks whether backups properly track & manage bulk files loads. + */ +@Tag(LargeTests.TAG) +public class TestIncrementalBackupWithBulkLoad extends TestBackupBase { + + private static final String TEST_NAME = TestIncrementalBackupWithBulkLoad.class.getSimpleName(); + private static final int ROWS_IN_BULK_LOAD = 100; + + // implement all test cases in 1 test since incremental backup/restore has dependencies + @Test + public void TestIncBackupDeleteTable() throws Exception { + try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + // The test starts with some data, and no bulk loaded rows. + int expectedRowCount = NB_ROWS_IN_BATCH; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertTrue(systemTable.readBulkloadRows(ImmutableList.of(table1)).isEmpty()); + + // Bulk loads aren't tracked if the table isn't backed up yet + performBulkLoad("bulk1"); + expectedRowCount += ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(0, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + + // Create a backup, bulk loads are now being tracked + String backup1 = backupTables(BackupType.FULL, ImmutableList.of(table1), BACKUP_ROOT_DIR); + assertTrue(checkSucceeded(backup1)); + performBulkLoad("bulk2"); + expectedRowCount += ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(1, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + + // Truncating or deleting a table clears the tracked bulk loads (and all rows) + TEST_UTIL.truncateTable(table1).close(); + expectedRowCount = 0; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(0, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + + // Creating a full backup clears the bulk loads (since they are captured in the snapshot) + performBulkLoad("bulk3"); + expectedRowCount = ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(1, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + String backup2 = backupTables(BackupType.FULL, ImmutableList.of(table1), BACKUP_ROOT_DIR); + assertTrue(checkSucceeded(backup2)); + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(0, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + + // Creating an incremental backup clears the bulk loads + performBulkLoad("bulk4"); + performBulkLoad("bulk5"); + performBulkLoad("bulk6"); + expectedRowCount += 3 * ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(3, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + String backup3 = + backupTables(BackupType.INCREMENTAL, ImmutableList.of(table1), BACKUP_ROOT_DIR); + assertTrue(checkSucceeded(backup3)); + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(0, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + int rowCountAfterBackup3 = expectedRowCount; + + // Doing another bulk load, to check that this data will disappear after a restore operation + performBulkLoad("bulk7"); + expectedRowCount += ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + List bulkloadsTemp = systemTable.readBulkloadRows(ImmutableList.of(table1)); + assertEquals(1, bulkloadsTemp.size()); + BulkLoad bulk7 = bulkloadsTemp.get(0); + + // Doing a restore. Overwriting the table implies clearing the bulk loads, + // but the loading of restored data involves loading bulk data, we expect 2 bulk loads + // associated with backup 3 (loading of full backup, loading of incremental backup). + BackupAdmin client = getBackupAdmin(); + client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backup3, false, + new TableName[] { table1 }, new TableName[] { table1 }, true)); + assertEquals(rowCountAfterBackup3, TEST_UTIL.countRows(table1)); + List bulkLoads = systemTable.readBulkloadRows(ImmutableList.of(table1)); + assertEquals(2, bulkLoads.size()); + assertFalse(bulkLoads.contains(bulk7)); + + // Check that we have data of all expected bulk loads + try (Table restoredTable = TEST_UTIL.getConnection().getTable(table1)) { + assertFalse(containsRowWithKey(restoredTable, "bulk1")); + assertFalse(containsRowWithKey(restoredTable, "bulk2")); + assertTrue(containsRowWithKey(restoredTable, "bulk3")); + assertTrue(containsRowWithKey(restoredTable, "bulk4")); + assertTrue(containsRowWithKey(restoredTable, "bulk5")); + assertTrue(containsRowWithKey(restoredTable, "bulk6")); + assertFalse(containsRowWithKey(restoredTable, "bulk7")); + } + } + } + + private boolean containsRowWithKey(Table table, String rowKey) throws IOException { + byte[] data = Bytes.toBytes(rowKey); + Get get = new Get(data); + Result result = table.get(get); + return result.containsColumn(famName, qualName); + } + + @Test + public void testUpdateFileListsRaceCondition() throws Exception { + try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + // Test the race condition where files are archived during incremental backup + FileSystem fs = TEST_UTIL.getTestFileSystem(); + + String regionName = "region1"; + String columnFamily = "cf"; + String filename1 = "hfile1"; + String filename2 = "hfile2"; + + Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration()); + Path tableDir = CommonFSUtils.getTableDir(rootDir, table1); + Path activeFile1 = + new Path(tableDir, regionName + Path.SEPARATOR + columnFamily + Path.SEPARATOR + filename1); + Path activeFile2 = + new Path(tableDir, regionName + Path.SEPARATOR + columnFamily + Path.SEPARATOR + filename2); + + fs.mkdirs(activeFile1.getParent()); + fs.create(activeFile1).close(); + fs.create(activeFile2).close(); + + List activeFiles = new ArrayList<>(); + activeFiles.add(activeFile1.toString()); + activeFiles.add(activeFile2.toString()); + List archiveFiles = new ArrayList<>(); + + Path archiveDir = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(), table1, + regionName, columnFamily); + Path archivedFile1 = new Path(archiveDir, filename1); + fs.mkdirs(archiveDir); + assertTrue(fs.rename(activeFile1, archivedFile1), "File should be moved to archive"); + + TestBackupBase.IncrementalTableBackupClientForTest client = + new TestBackupBase.IncrementalTableBackupClientForTest(TEST_UTIL.getConnection(), + "test_backup_id", + createBackupRequest(BackupType.INCREMENTAL, ImmutableList.of(table1), BACKUP_ROOT_DIR)); + + client.updateFileLists(activeFiles, archiveFiles); + + assertEquals(1, activeFiles.size(), "Only one file should remain in active files"); + assertEquals(activeFile2.toString(), activeFiles.get(0), + "File2 should still be in active files"); + assertEquals(1, archiveFiles.size(), "One file should be added to archive files"); + assertEquals(archivedFile1.toString(), archiveFiles.get(0), + "Archived file should have correct path"); + systemTable.finishBackupExclusiveOperation(); + } + + } + + @Test + public void testUpdateFileListsMissingArchivedFile() throws Exception { + try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + // Test that IOException is thrown when file doesn't exist in archive location + FileSystem fs = TEST_UTIL.getTestFileSystem(); + + String regionName = "region2"; + String columnFamily = "cf"; + String filename = "missing_file"; + + Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration()); + Path tableDir = CommonFSUtils.getTableDir(rootDir, table1); + Path activeFile = + new Path(tableDir, regionName + Path.SEPARATOR + columnFamily + Path.SEPARATOR + filename); + + fs.mkdirs(activeFile.getParent()); + fs.create(activeFile).close(); + + List activeFiles = new ArrayList<>(); + activeFiles.add(activeFile.toString()); + List archiveFiles = new ArrayList<>(); + + // Delete the file but don't create it in archive location + fs.delete(activeFile, false); + + TestBackupBase.IncrementalTableBackupClientForTest client = + new TestBackupBase.IncrementalTableBackupClientForTest(TEST_UTIL.getConnection(), + "test_backup_id", + createBackupRequest(BackupType.INCREMENTAL, ImmutableList.of(table1), BACKUP_ROOT_DIR)); + + // This should throw IOException since file doesn't exist in archive + try { + client.updateFileLists(activeFiles, archiveFiles); + fail("Expected IOException to be thrown"); + } catch (IOException e) { + // Expected + } + systemTable.finishBackupExclusiveOperation(); + } + } + + private void performBulkLoad(String keyPrefix) throws IOException { + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Path baseDirectory = TEST_UTIL.getDataTestDirOnTestFS(TEST_NAME); + Path hfilePath = + new Path(baseDirectory, Bytes.toString(famName) + Path.SEPARATOR + "hfile_" + keyPrefix); + + HFileTestUtil.createHFile(TEST_UTIL.getConfiguration(), fs, hfilePath, famName, qualName, + Bytes.toBytes(keyPrefix), Bytes.toBytes(keyPrefix + "z"), ROWS_IN_BULK_LOAD); + + Map result = + BulkLoadHFiles.create(TEST_UTIL.getConfiguration()).bulkLoad(table1, baseDirectory); + assertFalse(result.isEmpty()); + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithDataLoss.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithDataLoss.java index cf442f5f0dd7..efbe7cbe46a1 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithDataLoss.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithDataLoss.java @@ -17,31 +17,25 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestIncrementalBackupWithDataLoss extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIncrementalBackupWithDataLoss.class); - private static final Logger LOG = LoggerFactory.getLogger(TestIncrementalBackupWithDataLoss.class); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java index c8d536564188..b1dec43fef54 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java @@ -17,13 +17,11 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; @@ -41,37 +39,21 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.ToolRunner; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) -@RunWith(Parameterized.class) +@Tag(LargeTests.TAG) public class TestIncrementalBackupWithFailures extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIncrementalBackupWithFailures.class); - private static final Logger LOG = LoggerFactory.getLogger(TestIncrementalBackupWithFailures.class); - @Parameterized.Parameters - public static Collection data() { + static { provider = "multiwal"; - List params = new ArrayList(); - params.add(new Object[] { Boolean.TRUE }); - return params; - } - - public TestIncrementalBackupWithFailures(Boolean b) { } // implement all test cases in 1 test since incremental backup/restore has dependencies @@ -103,7 +85,7 @@ public void testIncBackupRestore() throws Exception { Table t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS); LOG.debug("writing " + ADD_ROWS + " rows to " + table1); - Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3); + assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3); t1.close(); LOG.debug("written " + ADD_ROWS + " rows to " + table1); @@ -115,7 +97,7 @@ public void testIncBackupRestore() throws Exception { t2.put(p2); } - Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + 5); + assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + 5); t2.close(); LOG.debug("written " + 5 + " rows to " + table2); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java index f9b9ab7ab500..bfa53567a01e 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.concurrent.CountDownLatch; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -38,30 +38,24 @@ import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestRemoteBackup extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRemoteBackup.class); - private static final Logger LOG = LoggerFactory.getLogger(TestRemoteBackup.class); /** * Setup Cluster with appropriate configurations before running tests. * @throws Exception if starting the mini cluster or setting up the tables fails */ - @BeforeClass + @BeforeAll public static void setUp() throws Exception { TEST_UTIL = new HBaseTestingUtility(); conf1 = TEST_UTIL.getConfiguration(); @@ -120,10 +114,10 @@ public void testFullBackupRemote() throws Exception { assertTrue(checkSucceeded(backupId)); LOG.info("backup complete " + backupId); - Assert.assertEquals(TEST_UTIL.countRows(t1, famName), NB_ROWS_IN_BATCH); + assertEquals(NB_ROWS_IN_BATCH, TEST_UTIL.countRows(t1, famName)); t.join(); - Assert.assertEquals(TEST_UTIL.countRows(t1, fam3Name), NB_ROWS_IN_FAM3); + assertEquals(NB_ROWS_IN_FAM3, TEST_UTIL.countRows(t1, fam3Name)); t1.close(); TableName[] tablesRestoreFull = new TableName[] { table1 }; @@ -140,12 +134,12 @@ public void testFullBackupRemote() throws Exception { // #5.2 - checking row count of tables for full restore Table hTable = conn.getTable(table1_restore); - Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH); + assertEquals(NB_ROWS_IN_BATCH, TEST_UTIL.countRows(hTable, famName)); int cnt3 = TEST_UTIL.countRows(hTable, fam3Name); - Assert.assertTrue(cnt3 >= 0 && cnt3 <= NB_ROWS_IN_FAM3); + assertTrue(cnt3 >= 0 && cnt3 <= NB_ROWS_IN_FAM3); int rows1 = MobSnapshotTestingUtils.countMobRows(t1, fam2Name); - Assert.assertEquals(rows0, rows1); + assertEquals(rows0, rows1); hTable.close(); hAdmin.close(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java index 53ef8955df3e..a46d5df2ce00 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java @@ -17,13 +17,12 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -33,27 +32,22 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestRemoteRestore extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRemoteRestore.class); - private static final Logger LOG = LoggerFactory.getLogger(TestRemoteRestore.class); /** * Setup Cluster with appropriate configurations before running tests. * @throws Exception if starting the mini cluster or setting up the tables fails */ - @BeforeClass + @BeforeAll public static void setUp() throws Exception { TEST_UTIL = new HBaseTestingUtility(); conf1 = TEST_UTIL.getConfiguration(); @@ -109,7 +103,7 @@ public void testFullRestoreRemoteWithAlternateRestoreOutputDir() throws Exceptio new BackupAdminImpl(ConnectionFactory.createConnection(conf)) .restore(new RestoreRequest.Builder().withBackupRootDir(BACKUP_REMOTE_ROOT_DIR) .withRestoreRootDir(BACKUP_ROOT_DIR).withBackupId(backupId).withCheck(false) - .withFromTables(tableset).withToTables(tablemap).withOvewrite(false).build()); + .withFromTables(tableset).withToTables(tablemap).withOverwrite(false).build()); Path hfileOutputPath = new Path( new Path(conf.get(MapReduceHFileSplitterJob.BULK_OUTPUT_CONF_KEY)).toUri().getPath()); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java index 93345fd17059..a134c974fee6 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRepairAfterFailedDelete.java @@ -17,33 +17,27 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestRepairAfterFailedDelete extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRepairAfterFailedDelete.class); - private static final Logger LOG = LoggerFactory.getLogger(TestRepairAfterFailedDelete.class); @Test diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java index 7b49558031e8..d935dd6ff6ab 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java @@ -17,27 +17,21 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestRestoreBoundaryTests extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRestoreBoundaryTests.class); - private static final Logger LOG = LoggerFactory.getLogger(TestRestoreBoundaryTests.class); /** diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java index f626dec5875d..40e078f5b576 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java @@ -17,30 +17,27 @@ */ package org.apache.hadoop.hbase.backup; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestSystemTableSnapshot extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSystemTableSnapshot.class); - private static final Logger LOG = LoggerFactory.getLogger(TestSystemTableSnapshot.class); /** * Verify backup system table snapshot. * @throws Exception if an operation on the table fails */ - // @Test + @Disabled + @Test public void _testBackupRestoreSystemTable() throws Exception { LOG.info("test snapshot system table"); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java index d84087990345..6f354b996def 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java @@ -19,25 +19,19 @@ import static org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyJob.BACKUP_COPY_OPTION_PREFIX; import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_MAX_MAPS; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestMapReduceBackupCopyJob { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMapReduceBackupCopyJob.class); - @Test public void testDistCpOptionParsing() { Configuration conf = new Configuration(); @@ -46,7 +40,7 @@ public void testDistCpOptionParsing() { List expectedArgs = ImmutableList. builder().add("-m", "1000").build(); - assertEquals(args, expectedArgs); + assertEquals(expectedArgs, args); } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java index 0602327f9bd3..3eaa1c633cd3 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.hbase.backup.master; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -31,35 +33,41 @@ import java.util.Set; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.backup.TestBackupBase; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.backup.util.BackupBoundaries; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(LargeTests.class) +@Tag(LargeTests.TAG) public class TestBackupLogCleaner extends TestBackupBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBackupLogCleaner.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBackupLogCleaner.class); // implements all test cases in 1 test since incremental full backup/ // incremental backup has dependencies + @BeforeAll + public static void before() { + TEST_UTIL.getConfiguration().setLong(BackupLogCleaner.TS_BUFFER_KEY, 0); + } + @Test public void testBackupLogCleaner() throws Exception { Path backupRoot1 = new Path(BACKUP_ROOT_DIR, "root1"); @@ -197,35 +205,188 @@ public void testBackupLogCleaner() throws Exception { // Taking the minimum timestamp (= 2), this means all WALs preceding B3 can be deleted. deletable = cleaner.getDeletableFiles(walFilesAfterB5); assertEquals(toSet(walFilesAfterB2), toSet(deletable)); + } finally { + TEST_UTIL.truncateTable(BackupSystemTable.getTableName(TEST_UTIL.getConfiguration())).close(); } } - private Set mergeAsSet(Collection toCopy, Collection toAdd) { - Set result = new LinkedHashSet<>(toCopy); - result.addAll(toAdd); - return result; + @Test + public void testDoesNotDeleteWALsFromNewServers() throws Exception { + Path backupRoot1 = new Path(BACKUP_ROOT_DIR, "backup1"); + List tableSetFull = Arrays.asList(table1, table2, table3, table4); + + JVMClusterUtil.RegionServerThread rsThread = null; + try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + LOG.info("Creating initial backup B1"); + String backupIdB1 = backupTables(BackupType.FULL, tableSetFull, backupRoot1.toString()); + assertTrue(checkSucceeded(backupIdB1)); + + List walsAfterB1 = getListOfWALFiles(TEST_UTIL.getConfiguration()); + LOG.info("WALs after B1: {}", walsAfterB1.size()); + + // Add a new RegionServer to the cluster + LOG.info("Adding new RegionServer to cluster"); + rsThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer(); + ServerName newServerName = rsThread.getRegionServer().getServerName(); + LOG.info("New RegionServer started: {}", newServerName); + + // Move a region to the new server to ensure it creates a WAL + List regions = TEST_UTIL.getAdmin().getRegions(table1); + RegionInfo regionToMove = regions.get(0); + + LOG.info("Moving region {} to new server {}", regionToMove.getEncodedName(), newServerName); + TEST_UTIL.getAdmin().move(regionToMove.getEncodedNameAsBytes(), newServerName); + + TEST_UTIL.waitFor(30000, () -> { + try { + HRegionLocation location = TEST_UTIL.getConnection().getRegionLocator(table1) + .getRegionLocation(regionToMove.getStartKey()); + return location.getServerName().equals(newServerName); + } catch (IOException e) { + return false; + } + }); + + // Write some data to trigger WAL creation on the new server + try (Table t1 = TEST_UTIL.getConnection().getTable(table1)) { + for (int i = 0; i < 100; i++) { + Put p = new Put(Bytes.toBytes("newserver-row-" + i)); + p.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t1.put(p); + } + } + TEST_UTIL.getAdmin().flushRegion(regionToMove.getEncodedNameAsBytes()); + + List walsAfterNewServer = getListOfWALFiles(TEST_UTIL.getConfiguration()); + LOG.info("WALs after adding new server: {}", walsAfterNewServer.size()); + assertTrue(walsAfterNewServer.size() > walsAfterB1.size(), + "Should have more WALs after new server"); + + List newServerWALs = new ArrayList<>(walsAfterNewServer); + newServerWALs.removeAll(walsAfterB1); + assertFalse(newServerWALs.isEmpty(), "Should have WALs from new server"); + + BackupLogCleaner cleaner = new BackupLogCleaner(); + cleaner.setConf(TEST_UTIL.getConfiguration()); + Map params = new HashMap<>(1); + params.put(HMaster.MASTER, TEST_UTIL.getHBaseCluster().getMaster()); + cleaner.init(params); + + Set deletable = toSet(cleaner.getDeletableFiles(walsAfterNewServer)); + for (FileStatus newWAL : newServerWALs) { + assertFalse(deletable.contains(newWAL), + "WAL from new server should NOT be deletable: " + newWAL.getPath()); + } + } finally { + TEST_UTIL.truncateTable(BackupSystemTable.getTableName(TEST_UTIL.getConfiguration())).close(); + // Clean up the RegionServer we added + if (rsThread != null) { + LOG.info("Stopping the RegionServer added for test"); + TEST_UTIL.getMiniHBaseCluster() + .stopRegionServer(rsThread.getRegionServer().getServerName()); + TEST_UTIL.getMiniHBaseCluster() + .waitForRegionServerToStop(rsThread.getRegionServer().getServerName(), 30000); + } + } } - private Set toSet(Iterable iterable) { - Set result = new LinkedHashSet<>(); - iterable.forEach(result::add); - return result; + @Test + public void testCanDeleteFileWithNewServerWALs() { + BackupInfo backup = new BackupInfo(); + backup.setState(BackupInfo.BackupState.COMPLETE); + backup.setTableSetTimestampMap(Collections.singletonMap(TableName.valueOf("table1"), + Collections.singletonMap("server1:60020", 1000000L))); + BackupBoundaries boundaries = + BackupLogCleaner.calculatePreservationBoundary(Collections.singletonList(backup), 0L); + + // Old WAL from before the backup + Path oldWAL = new Path("/hbase/oldWALs/server1%2C60020%2C12345.500000"); + assertTrue(BackupLogCleaner.canDeleteFile(boundaries, oldWAL), + "WAL older than backup should be deletable"); + + // WAL from exactly at the backup boundary + Path boundaryWAL = new Path("/hbase/oldWALs/server1%2C60020%2C12345.1000000"); + assertTrue(BackupLogCleaner.canDeleteFile(boundaries, boundaryWAL), + "WAL at boundary should be deletable"); + + // WAL created after the backup boundary + Path newWal = new Path("/hbase/oldWALs/server1%2C60020%2C12345.1500000"); + assertFalse(BackupLogCleaner.canDeleteFile(boundaries, newWal), + "WAL newer than backup should not be deletable"); + + // WAL from a new server that joined AFTER the backup + Path newServerWAL = new Path("/hbase/oldWALs/newserver%2C60020%2C99999.1500000"); + assertFalse(BackupLogCleaner.canDeleteFile(boundaries, newServerWAL), + "WAL from new server (after backup) should NOT be deletable"); + } + + @Test + public void testFirstBackupProtectsFiles() { + BackupInfo backup = new BackupInfo(); + backup.setBackupId("backup_1"); + backup.setState(BackupInfo.BackupState.RUNNING); + backup.setStartTs(100L); + // Running backups have no TableSetTimestampMap + + BackupBoundaries boundaries = + BackupLogCleaner.calculatePreservationBoundary(Collections.singletonList(backup), 5L); + + // There's only a single backup, and it is still running, so it's a FULL backup. + // We expect files preceding the snapshot are deletable, but files after the start are not. + // Because this is not region-server-specific, the buffer is taken into account. + Path path = new Path("/hbase/oldWALs/server1%2C60020%2C12345.94"); + assertTrue(BackupLogCleaner.canDeleteFile(boundaries, path)); + path = new Path("/hbase/oldWALs/server1%2C60020%2C12345.95"); + assertTrue(BackupLogCleaner.canDeleteFile(boundaries, path)); + path = new Path("/hbase/oldWALs/server1%2C60020%2C12345.96"); + assertFalse(BackupLogCleaner.canDeleteFile(boundaries, path)); + + // If there is an already completed backup in the same root, only that one matters. + // In this case, a region-server-specific timestamp is available, so the buffer is not used. + BackupInfo backup2 = new BackupInfo(); + backup2.setBackupId("backup_2"); + backup2.setState(BackupInfo.BackupState.COMPLETE); + backup2.setStartTs(80L); + backup2.setTableSetTimestampMap(Collections.singletonMap(TableName.valueOf("table1"), + Collections.singletonMap("server1:60020", 90L))); + + boundaries = BackupLogCleaner.calculatePreservationBoundary(Arrays.asList(backup, backup2), 5L); + + path = new Path("/hbase/oldWALs/server1%2C60020%2C12345.89"); + assertTrue(BackupLogCleaner.canDeleteFile(boundaries, path)); + path = new Path("/hbase/oldWALs/server1%2C60020%2C12345.90"); + assertTrue(BackupLogCleaner.canDeleteFile(boundaries, path)); + path = new Path("/hbase/oldWALs/server1%2C60020%2C12345.91"); + assertFalse(BackupLogCleaner.canDeleteFile(boundaries, path)); } @Test public void testCleansUpHMasterWal() { Path path = new Path("/hbase/MasterData/WALs/hmaster,60000,1718808578163"); - assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), path)); + assertTrue(BackupLogCleaner.canDeleteFile(BackupBoundaries.builder(0L).build(), path)); } @Test public void testCleansUpArchivedHMasterWal() { + BackupBoundaries empty = BackupBoundaries.builder(0L).build(); Path normalPath = new Path("/hbase/oldWALs/hmaster%2C60000%2C1716224062663.1716247552189$masterlocalwal$"); - assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), normalPath)); + assertTrue(BackupLogCleaner.canDeleteFile(empty, normalPath)); Path masterPath = new Path( "/hbase/MasterData/oldWALs/hmaster%2C60000%2C1716224062663.1716247552189$masterlocalwal$"); - assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), masterPath)); + assertTrue(BackupLogCleaner.canDeleteFile(empty, masterPath)); + } + + private Set mergeAsSet(Collection toCopy, Collection toAdd) { + Set result = new LinkedHashSet<>(toCopy); + result.addAll(toAdd); + return result; + } + + private Set toSet(Iterable iterable) { + Set result = new LinkedHashSet<>(); + iterable.forEach(result::add); + return result; } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestRestoreBackupSystemTable.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestRestoreBackupSystemTable.java new file mode 100644 index 000000000000..6ea984f72bf2 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestRestoreBackupSystemTable.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.master; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestRestoreBackupSystemTable { + private static final String BACKUP_ROOT = "root"; + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.startMiniCluster(); + } + + @Test + public void itRestoresFromSnapshot() throws Exception { + BackupSystemTable table = new BackupSystemTable(UTIL.getConnection()); + Set tables = new HashSet<>(); + + tables.add(TableName.valueOf("test1")); + tables.add(TableName.valueOf("test2")); + tables.add(TableName.valueOf("test3")); + + Map rsTimestampMap = new HashMap<>(); + rsTimestampMap.put("rs1:100", 100L); + rsTimestampMap.put("rs2:100", 101L); + rsTimestampMap.put("rs3:100", 103L); + + table.writeRegionServerLogTimestamp(tables, rsTimestampMap, BACKUP_ROOT); + BackupSystemTable.snapshot(UTIL.getConnection()); + + Admin admin = UTIL.getAdmin(); + TableName backupSystemTn = BackupSystemTable.getTableName(UTIL.getConfiguration()); + admin.disableTable(backupSystemTn); + admin.truncateTable(backupSystemTn, true); + + BackupSystemTable.restoreFromSnapshot(UTIL.getConnection()); + Map> results = table.readLogTimestampMap(BACKUP_ROOT); + + assertEquals(results.size(), tables.size()); + + for (TableName tableName : tables) { + Map resultMap = results.get(tableName); + assertEquals(resultMap, rsTimestampMap); + } + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } +} diff --git a/hbase-checkstyle/pom.xml b/hbase-checkstyle/pom.xml index bd8d5f7e80bb..67fc77a4dc5f 100644 --- a/hbase-checkstyle/pom.xml +++ b/hbase-checkstyle/pom.xml @@ -43,6 +43,13 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + true + + org.apache.maven.plugins maven-site-plugin diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml index 2d8e880ddf4a..36b242fd9fa6 100644 --- a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml +++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml @@ -32,7 +32,6 @@ --> - diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index 8343037a233d..e54ca13d63cd 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -89,6 +89,12 @@ compile true + + com.github.stephenc.jcip + jcip-annotations + compile + true + commons-codec commons-codec @@ -173,13 +179,23 @@ metrics-core - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test org.mockito - mockito-core + mockito-junit-jupiter test diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 2ef70daab0a8..1819dda048dc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -285,6 +285,7 @@ public static HRegionLocation getRegionLocation(Connection connection, byte[] re } Get get = new Get(row); get.addFamily(HConstants.CATALOG_FAMILY); + get.setPriority(HConstants.INTERNAL_READ_QOS); Result r = get(getMetaHTable(connection), get); RegionLocations locations = getRegionLocations(r); return locations == null @@ -310,6 +311,7 @@ public static Result getCatalogFamilyRow(Connection connection, RegionInfo ri) throws IOException { Get get = new Get(getMetaKeyForRegion(ri)); get.addFamily(HConstants.CATALOG_FAMILY); + get.setPriority(HConstants.INTERNAL_READ_QOS); return get(getMetaHTable(connection), get); } @@ -339,9 +341,7 @@ public static RegionInfo parseRegionInfoFromRegionName(byte[] regionName) throws */ public static Result getRegionResult(Connection connection, RegionInfo regionInfo) throws IOException { - Get get = new Get(getMetaKeyForRegion(regionInfo)); - get.addFamily(HConstants.CATALOG_FAMILY); - return get(getMetaHTable(connection), get); + return getCatalogFamilyRow(connection, regionInfo); } /** @@ -576,6 +576,7 @@ private static Scan getMetaScan(Configuration conf, int rowUpperLimit) { scan.setReadType(Scan.ReadType.PREAD); } scan.setCaching(scannerCaching); + scan.setPriority(HConstants.INTERNAL_READ_QOS); return scan; } @@ -776,9 +777,11 @@ private static void scanMeta(Connection connection, @Nullable final byte[] start } if (LOG.isTraceEnabled()) { - LOG.trace("Scanning META" + " starting at row=" + Bytes.toStringBinary(startRow) - + " stopping at row=" + Bytes.toStringBinary(stopRow) + " for max=" + rowUpperLimit - + " with caching=" + scan.getCaching()); + LOG.trace( + "Scanning META starting at row={} stopping at row={} for max={} with caching={} " + + "priority={}", + Bytes.toStringBinary(startRow), Bytes.toStringBinary(stopRow), rowUpperLimit, + scan.getCaching(), scan.getPriority()); } int currentRow = 0; @@ -1774,7 +1777,7 @@ private static void updateLocation(Connection connection, RegionInfo regionInfo, addRegionInfo(put, regionInfo); addLocation(put, sn, openSeqNum, regionInfo.getReplicaId()); putToMetaTable(connection, put); - LOG.info("Updated row {} with server=", regionInfo.getRegionNameAsString(), sn); + LOG.info("Updated row {} with server = {}", regionInfo.getRegionNameAsString(), sn); } /** @@ -1899,7 +1902,7 @@ public static Put addLocation(Put p, ServerName sn, long openSeqNum, int replica .setType(Cell.Type.Put).setValue(Bytes.toBytes(sn.getAddress().toString())).build()) .add(builder.clear().setRow(p.getRow()).setFamily(getCatalogFamily()) .setQualifier(getStartCodeColumn(replicaId)).setTimestamp(p.getTimestamp()) - .setType(Cell.Type.Put).setValue(Bytes.toBytes(sn.getStartcode())).build()) + .setType(Cell.Type.Put).setValue(Bytes.toBytes(sn.getStartCode())).build()) .add(builder.clear().setRow(p.getRow()).setFamily(getCatalogFamily()) .setQualifier(getSeqNumColumn(replicaId)).setTimestamp(p.getTimestamp()) .setType(Cell.Type.Put).setValue(Bytes.toBytes(openSeqNum)).build()); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index c341cf4a9895..ccbbf9a08e48 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1618,6 +1618,47 @@ default Future modifyTableAsync(TableDescriptor td) throws IOException { */ Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) throws IOException; + /** + * Reopen all regions of a table. This is useful after calling + * {@link #modifyTableAsync(TableDescriptor, boolean)} with reopenRegions=false to gradually roll + * out table descriptor changes to regions. Regions are reopened in-place (no move). + * @param tableName table whose regions to reopen + * @throws IOException if a remote or network exception occurs + */ + default void reopenTableRegions(TableName tableName) throws IOException { + get(reopenTableRegionsAsync(tableName), getSyncWaitTimeout(), TimeUnit.MILLISECONDS); + } + + /** + * Reopen specific regions of a table. Useful for canary testing table descriptor changes on a + * subset of regions before rolling out to the entire table. + * @param tableName table whose regions to reopen + * @param regions specific regions to reopen + * @throws IOException if a remote or network exception occurs + */ + default void reopenTableRegions(TableName tableName, List regions) + throws IOException { + get(reopenTableRegionsAsync(tableName, regions), getSyncWaitTimeout(), TimeUnit.MILLISECONDS); + } + + /** + * Asynchronously reopen all regions of a table. + * @param tableName table whose regions to reopen + * @return Future for tracking completion + * @throws IOException if a remote or network exception occurs + */ + Future reopenTableRegionsAsync(TableName tableName) throws IOException; + + /** + * Asynchronously reopen specific regions of a table. + * @param tableName table whose regions to reopen + * @param regions specific regions to reopen + * @return Future for tracking completion + * @throws IOException if a remote or network exception occurs + */ + Future reopenTableRegionsAsync(TableName tableName, List regions) + throws IOException; + /** * Change the store file tracker of the given table. * @param tableName the table you want to change @@ -3356,4 +3397,15 @@ List getLogEntries(Set serverNames, String logType, Server * Get the list of cached files */ List getCachedFilesList(ServerName serverName) throws IOException; + + @InterfaceAudience.Private + default void restoreBackupSystemTable(String snapshotName) throws IOException { + SnapshotDescription snapshot = + listSnapshots().stream().filter(s -> s.getName().equals(snapshotName)).findFirst() + .orElseThrow(() -> new IOException("Snapshot " + snapshotName + " not found")); + TableName tn = snapshot.getTableName(); + disableTable(tn); + restoreSnapshot(snapshotName); + enableTable(tn); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index ae651c3e0074..6b38c13547df 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -208,6 +208,24 @@ default CompletableFuture modifyTable(TableDescriptor desc) { */ CompletableFuture modifyTable(TableDescriptor desc, boolean reopenRegions); + /** + * Reopen all regions of a table. This is useful after calling + * {@link #modifyTable(TableDescriptor, boolean)} with reopenRegions=false to gradually roll out + * table descriptor changes to regions. Regions are reopened in-place (no move). + * @param tableName table whose regions to reopen + * @return CompletableFuture that completes when all regions have been reopened + */ + CompletableFuture reopenTableRegions(TableName tableName); + + /** + * Reopen specific regions of a table. Useful for canary testing table descriptor changes on a + * subset of regions before rolling out to the entire table. + * @param tableName table whose regions to reopen + * @param regions specific regions to reopen + * @return CompletableFuture that completes when specified regions have been reopened + */ + CompletableFuture reopenTableRegions(TableName tableName, List regions); + /** * Change the store file tracker of the given table. * @param tableName the table you want to change @@ -1721,4 +1739,7 @@ CompletableFuture> getLogEntries(Set serverNames, Str * Get the list of cached files */ CompletableFuture> getCachedFilesList(ServerName serverName); + + @InterfaceAudience.Private + CompletableFuture restoreBackupSystemTable(String snapshotName); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBufferedMutatorImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBufferedMutatorImpl.java index e5500b0977b1..6d41d28c852e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBufferedMutatorImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBufferedMutatorImpl.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.client; -import static org.apache.hadoop.hbase.client.ConnectionUtils.validatePut; +import static org.apache.hadoop.hbase.client.ConnectionUtils.validateMutation; import static org.apache.hadoop.hbase.util.FutureUtils.addListener; import java.io.IOException; @@ -27,8 +27,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.yetus.audience.InterfaceAudience; @@ -46,6 +45,18 @@ class AsyncBufferedMutatorImpl implements AsyncBufferedMutator { private static final Logger LOG = LoggerFactory.getLogger(AsyncBufferedMutatorImpl.class); + private static final int INITIAL_CAPACITY = 100; + + protected static class Batch { + final ArrayList toSend; + final ArrayList> toComplete; + + Batch(ArrayList toSend, ArrayList> toComplete) { + this.toSend = toSend; + this.toComplete = toComplete; + } + } + private final HashedWheelTimer periodicalFlushTimer; private final AsyncTable table; @@ -58,16 +69,20 @@ class AsyncBufferedMutatorImpl implements AsyncBufferedMutator { private final int maxMutations; - private List mutations = new ArrayList<>(); + private ArrayList mutations = new ArrayList<>(INITIAL_CAPACITY); - private List> futures = new ArrayList<>(); + private ArrayList> futures = new ArrayList<>(INITIAL_CAPACITY); private long bufferedSize; - private boolean closed; + private volatile boolean closed; + // Accessed by tests Timeout periodicFlushTask; + // Accessed by tests + final ReentrantLock lock = new ReentrantLock(); + AsyncBufferedMutatorImpl(HashedWheelTimer periodicalFlushTimer, AsyncTable table, long writeBufferSize, long periodicFlushTimeoutNs, int maxKeyValueSize, int maxMutations) { this.periodicalFlushTimer = periodicalFlushTimer; @@ -88,23 +103,53 @@ public Configuration getConfiguration() { return table.getConfiguration(); } - // will be overridden in test - protected void internalFlush() { + /** + * Atomically drains the current buffered mutations and futures under {@link #lock} and prepares + * this mutator to accept a new batch. + *

+ * The {@link #lock} must be acquired before calling this method. Cancels any pending + * {@link #periodicFlushTask} to avoid a redundant flush for the data we are about to send. Swaps + * the shared {@link #mutations} and {@link #futures} lists into a returned {@link Batch}, + * replaces them with fresh lists, and resets {@link #bufferedSize} to zero. + *

+ * If there is nothing buffered, returns {@code null} so callers can skip sending work. + *

+ * Protected for being overridden in tests. + * @return a {@link Batch} containing drained mutations and futures, or {@code null} if empty + */ + protected Batch drainBatch() { + ArrayList toSend; + ArrayList> toComplete; + // Cancel the flush task if it is pending. if (periodicFlushTask != null) { periodicFlushTask.cancel(); periodicFlushTask = null; } - List toSend = this.mutations; + toSend = this.mutations; if (toSend.isEmpty()) { - return; + return null; } - List> toComplete = this.futures; + toComplete = this.futures; assert toSend.size() == toComplete.size(); - this.mutations = new ArrayList<>(); - this.futures = new ArrayList<>(); + this.mutations = new ArrayList<>(INITIAL_CAPACITY); + this.futures = new ArrayList<>(INITIAL_CAPACITY); bufferedSize = 0L; - Iterator> toCompleteIter = toComplete.iterator(); - for (CompletableFuture future : table.batch(toSend)) { + return new Batch(toSend, toComplete); + } + + /** + * Sends a previously drained {@link Batch} and wires the user-visible completion futures to the + * underlying results returned by {@link AsyncTable#batch(List)}. + *

+ * Preserves the one-to-one, in-order mapping between mutations and their corresponding futures. + * @param batch the drained batch to send; may be {@code null} + */ + private void sendBatch(Batch batch) { + if (batch == null) { + return; + } + Iterator> toCompleteIter = batch.toComplete.iterator(); + for (CompletableFuture future : table.batch(batch.toSend)) { CompletableFuture toCompleteFuture = toCompleteIter.next(); addListener(future, (r, e) -> { if (e != null) { @@ -118,33 +163,41 @@ protected void internalFlush() { @Override public List> mutate(List mutations) { - List> futures = - Stream.> generate(CompletableFuture::new).limit(mutations.size()) - .collect(Collectors.toList()); + List> futures = new ArrayList<>(mutations.size()); + for (int i = 0, n = mutations.size(); i < n; i++) { + futures.add(new CompletableFuture<>()); + } + if (closed) { + IOException ioe = new IOException("Already closed"); + futures.forEach(f -> f.completeExceptionally(ioe)); + return futures; + } long heapSize = 0; for (Mutation mutation : mutations) { heapSize += mutation.heapSize(); - if (mutation instanceof Put) { - validatePut((Put) mutation, maxKeyValueSize); - } + validateMutation(mutation, maxKeyValueSize); } - synchronized (this) { - if (closed) { - IOException ioe = new IOException("Already closed"); - futures.forEach(f -> f.completeExceptionally(ioe)); - return futures; - } + Batch batch = null; + lock.lock(); + try { if (this.mutations.isEmpty() && periodicFlushTimeoutNs > 0) { periodicFlushTask = periodicalFlushTimer.newTimeout(timeout -> { - synchronized (AsyncBufferedMutatorImpl.this) { - // confirm that we are still valid, if there is already an internalFlush call before us, - // then we should not execute any more. And in internalFlush we will set periodicFlush + Batch flushBatch = null; + lock.lock(); + try { + // confirm that we are still valid, if there is already a drainBatch call before us, + // then we should not execute anymore. And in drainBatch we will set periodicFlush // to null, and since we may schedule a new one, so here we check whether the references // are equal. if (timeout == periodicFlushTask) { periodicFlushTask = null; - internalFlush(); + flushBatch = drainBatch(); // Drains under lock } + } finally { + lock.unlock(); + } + if (flushBatch != null) { + sendBatch(flushBatch); // Sends outside of lock } }, periodicFlushTimeoutNs, TimeUnit.NANOSECONDS); } @@ -153,24 +206,55 @@ Stream.> generate(CompletableFuture::new).limit(mutation bufferedSize += heapSize; if (bufferedSize >= writeBufferSize) { LOG.trace("Flushing because write buffer size {} reached", writeBufferSize); - internalFlush(); + // drain now and send after releasing the lock + batch = drainBatch(); } else if (maxMutations > 0 && this.mutations.size() >= maxMutations) { LOG.trace("Flushing because max mutations {} reached", maxMutations); - internalFlush(); + batch = drainBatch(); } + } finally { + lock.unlock(); + } + // Send outside of lock + if (batch != null) { + sendBatch(batch); } return futures; } + // The only difference bewteen flush and close is that, we will set closed to true before sending + // out the batch to prevent further flush or close + private void flushOrClose(boolean close) { + Batch batch = null; + if (!closed) { + lock.lock(); + try { + if (!closed) { + // Drains under lock + batch = drainBatch(); + if (close) { + closed = true; + } + } + } finally { + lock.unlock(); + } + } + // Send the batch + if (batch != null) { + // Sends outside of lock + sendBatch(batch); + } + } + @Override - public synchronized void flush() { - internalFlush(); + public void flush() { + flushOrClose(false); } @Override - public synchronized void close() { - internalFlush(); - closed = true; + public void close() { + flushOrClose(true); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClientScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClientScanner.java index d58c8e60c8d8..8d3bcb000bc3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClientScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClientScanner.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.HConstants.EMPTY_START_ROW; import static org.apache.hadoop.hbase.client.ConnectionUtils.createScanResultCache; import static org.apache.hadoop.hbase.client.ConnectionUtils.getLocateType; +import static org.apache.hadoop.hbase.client.ConnectionUtils.incMillisBetweenNextsMetrics; import static org.apache.hadoop.hbase.client.ConnectionUtils.incRPCCallsMetrics; import static org.apache.hadoop.hbase.client.ConnectionUtils.incRPCRetriesMetrics; import static org.apache.hadoop.hbase.client.ConnectionUtils.incRegionCountMetrics; @@ -202,40 +203,42 @@ private CompletableFuture callOpenScanner(HBaseRpcControlle } } - private void startScan(OpenScannerResponse resp) { - addListener( + // lastNextCallNanos is used to calculate the MILLIS_BETWEEN_NEXTS scan metrics + private void startScan(OpenScannerResponse resp, long lastNextCallNanos) { + AsyncScanSingleRegionRpcRetryingCaller scanSingleRegionCaller = conn.callerFactory.scanSingleRegion().id(resp.resp.getScannerId()).location(resp.loc) .remote(resp.isRegionServerRemote) .scannerLeaseTimeoutPeriod(resp.resp.getTtl(), TimeUnit.MILLISECONDS).stub(resp.stub) .setScan(scan).metrics(scanMetrics).consumer(consumer).resultCache(resultCache) .rpcTimeout(rpcTimeoutNs, TimeUnit.NANOSECONDS) .scanTimeout(scanTimeoutNs, TimeUnit.NANOSECONDS).pause(pauseNs, TimeUnit.NANOSECONDS) + .lastNextCallNanos(lastNextCallNanos) .pauseForServerOverloaded(pauseNsForServerOverloaded, TimeUnit.NANOSECONDS) .maxAttempts(maxAttempts).startLogErrorsCnt(startLogErrorsCnt) - .setRequestAttributes(requestAttributes).start(resp.controller, resp.resp), - (hasMore, error) -> { - try (Scope ignored = span.makeCurrent()) { - if (error != null) { - try { - consumer.onError(error); - return; - } finally { - TraceUtil.setError(span, error); - span.end(); - } + .setRequestAttributes(requestAttributes).build(); + addListener(scanSingleRegionCaller.start(resp.controller, resp.resp), (hasMore, error) -> { + try (Scope ignored = span.makeCurrent()) { + if (error != null) { + try { + consumer.onError(error); + return; + } finally { + TraceUtil.setError(span, error); + span.end(); } - if (hasMore) { - openScanner(); - } else { - try { - consumer.onComplete(); - } finally { - span.setStatus(StatusCode.OK); - span.end(); - } + } + if (hasMore) { + openScanner(scanSingleRegionCaller); + } else { + try { + consumer.onComplete(); + } finally { + span.setStatus(StatusCode.OK); + span.end(); } } - }); + } + }); } private CompletableFuture openScanner(int replicaId) { @@ -256,11 +259,17 @@ private long getPrimaryTimeoutNs() { : conn.connConf.getPrimaryScanTimeoutNs(); } - private void openScanner() { + private void openScanner(AsyncScanSingleRegionRpcRetryingCaller previousScanSingleRegionCaller) { if (this.isScanMetricsByRegionEnabled) { scanMetrics.moveToNextRegion(); } incRegionCountMetrics(scanMetrics); + long openScannerStartNanos = System.nanoTime(); + if (previousScanSingleRegionCaller != null) { + // open scanner is also a next call + incMillisBetweenNextsMetrics(scanMetrics, TimeUnit.NANOSECONDS + .toMillis(openScannerStartNanos - previousScanSingleRegionCaller.getLastNextCallNanos())); + } openScannerTries.set(1); addListener(timelineConsistentRead(conn.getLocator(), tableName, scan, scan.getStartRow(), getLocateType(scan), this::openScanner, rpcTimeoutNs, getPrimaryTimeoutNs(), retryTimer, @@ -280,14 +289,14 @@ private void openScanner() { this.scanMetrics.initScanMetricsRegionInfo(loc.getRegion().getEncodedName(), loc.getServerName()); } - startScan(resp); + startScan(resp, openScannerStartNanos); } }); } public void start() { try (Scope ignored = span.makeCurrent()) { - openScanner(); + openScanner(null); } } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 448cfc9c36ee..fc7d5758e14e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -156,6 +156,16 @@ public CompletableFuture modifyTable(TableDescriptor desc, boolean reopenR return wrap(rawAdmin.modifyTable(desc, reopenRegions)); } + @Override + public CompletableFuture reopenTableRegions(TableName tableName) { + return wrap(rawAdmin.reopenTableRegions(tableName)); + } + + @Override + public CompletableFuture reopenTableRegions(TableName tableName, List regions) { + return wrap(rawAdmin.reopenTableRegions(tableName, regions)); + } + @Override public CompletableFuture modifyTableStoreFileTracker(TableName tableName, String dstSFT) { return wrap(rawAdmin.modifyTableStoreFileTracker(tableName, dstSFT)); @@ -921,4 +931,9 @@ public CompletableFuture flushMasterStore() { public CompletableFuture> getCachedFilesList(ServerName serverName) { return wrap(rawAdmin.getCachedFilesList(serverName)); } + + @Override + public CompletableFuture restoreBackupSystemTable(String snapshotName) { + return wrap(rawAdmin.restoreBackupSystemTable(snapshotName)); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java index 32776dde3e65..eb041a00f501 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -34,6 +35,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseServerException; import org.apache.hadoop.hbase.HConstants; @@ -52,6 +54,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Strings; + /** * The context, and return value, for a single submit/submitAll call. Note on how this class (one AP * submit) works. Initially, all requests are split into groups by server; request is sent to each @@ -195,7 +199,7 @@ public void run() { try { // setup the callable based on the actions, if we don't have one already from the request if (callable == null) { - callable = createCallable(server, tableName, multiAction); + callable = createCallable(server, tableName, multiAction, numAttempt); } RpcRetryingCaller caller = asyncProcess.createCaller(callable, rpcTimeout); @@ -219,13 +223,14 @@ public void run() { } catch (IOException e) { // The service itself failed . It may be an error coming from the communication // layer, but, as well, a functional error raised by the server. - receiveGlobalFailure(multiAction, server, numAttempt, e, true); + + receiveGlobalFailure(multiAction, server, numAttempt, e); return; } catch (Throwable t) { // This should not happen. Let's log & retry anyway. LOG.error("id=" + asyncProcess.id + ", caught throwable. Unexpected." + " Retrying. Server=" + server + ", tableName=" + tableName, t); - receiveGlobalFailure(multiAction, server, numAttempt, t, true); + receiveGlobalFailure(multiAction, server, numAttempt, t); return; } if (res.type() == AbstractResponse.ResponseType.MULTI) { @@ -387,10 +392,8 @@ public AsyncRequestFutureImpl(AsyncProcessTask task, List actions, long } else { this.replicaGetIndices = null; } - this.callsInProgress = !hasAnyReplicaGets - ? null - : Collections - .newSetFromMap(new ConcurrentHashMap()); + this.callsInProgress = + Collections.newSetFromMap(new ConcurrentHashMap()); this.asyncProcess = asyncProcess; this.errorsByServer = createServerErrorTracker(); this.errors = new BatchErrors(); @@ -536,7 +539,12 @@ private HRegionLocation getReplicaLocationOrFail(Action action) { private void manageLocationError(Action action, Exception ex) { String msg = - "Cannot get replica " + action.getReplicaId() + " location for " + action.getAction(); + "Cannot get replica " + action.getReplicaId() + " location for " + action.getAction() + ": "; + if (ex instanceof OperationTimeoutExceededException) { + msg += "Operation timeout exceeded."; + } else { + msg += ex == null ? "null cause" : ex.toString(); + } LOG.error(msg); if (ex == null) { ex = new IOException(msg); @@ -599,7 +607,6 @@ private void failIncompleteActionsWithOpTimeout(List actions, */ void sendMultiAction(Map actionsByServer, int numAttempt, List actionsForReplicaThread, boolean reuseThread) { - boolean clearServerCache = true; // Run the last item on the same thread if we are already on a send thread. // We hope most of the time it will be the only item, so we can cut down on threads. int actionsRemaining = actionsByServer.size(); @@ -635,7 +642,6 @@ void sendMultiAction(Map actionsByServer, int numAttemp LOG.warn("id=" + asyncProcess.id + ", task rejected by pool. Unexpected." + " Server=" + server.getServerName(), t); // Do not update cache if exception is from failing to submit action to thread pool - clearServerCache = false; } else { // see #HBASE-14359 for more details LOG.warn("Caught unexpected exception/error: ", t); @@ -643,7 +649,7 @@ void sendMultiAction(Map actionsByServer, int numAttemp asyncProcess.decTaskCounters(multiAction.getRegions(), server); // We're likely to fail again, but this will increment the attempt counter, // so it will finish. - receiveGlobalFailure(multiAction, server, numAttempt, t, clearServerCache); + receiveGlobalFailure(multiAction, server, numAttempt, t); } } } @@ -793,13 +799,24 @@ private void failAll(MultiAction actions, ServerName server, int numAttempt, * @param t the throwable (if any) that caused the resubmit */ private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int numAttempt, - Throwable t, boolean clearServerCache) { + Throwable t) { errorsByServer.reportServerError(server); Retry canRetry = errorsByServer.canTryMore(numAttempt) ? Retry.YES : Retry.NO_RETRIES_EXHAUSTED; + boolean clearServerCache; + + if (t instanceof RejectedExecutionException) { + clearServerCache = false; + } else { + clearServerCache = ClientExceptionsUtil.isMetaClearingException(t); + } // Do not update cache if exception is from failing to submit action to thread pool if (clearServerCache) { cleanServerCache(server, t); + + if (LOG.isTraceEnabled()) { + LOG.trace("Cleared meta cache for server {} due to global failure {}", server, t); + } } int failed = 0; @@ -808,12 +825,8 @@ private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int for (Map.Entry> e : rsActions.actions.entrySet()) { byte[] regionName = e.getKey(); byte[] row = e.getValue().get(0).getAction().getRow(); - // Do not use the exception for updating cache because it might be coming from - // any of the regions in the MultiAction and do not update cache if exception is - // from failing to submit action to thread pool if (clearServerCache) { - updateCachedLocations(server, regionName, row, - ClientExceptionsUtil.isMetaClearingException(t) ? null : t); + updateCachedLocations(server, regionName, row, t); } for (Action action : e.getValue()) { Retry retry = @@ -1276,20 +1289,31 @@ private String buildDetailedErrorMsg(String string, int index) { @Override public void waitUntilDone() throws InterruptedIOException { + long startTime = EnvironmentEdgeManager.currentTime(); try { if (this.operationTimeout > 0) { // the worker thread maybe over by some exception without decrement the actionsInProgress, // then the guarantee of operationTimeout will be broken, so we should set cutoff to avoid // stuck here forever - long cutoff = (EnvironmentEdgeManager.currentTime() + this.operationTimeout) * 1000L; + long cutoff = (startTime + this.operationTimeout) * 1000L; if (!waitUntilDone(cutoff)) { - throw new SocketTimeoutException("time out before the actionsInProgress changed to zero"); + String msg = "time out before the actionsInProgress changed to zero, with " + + actionsInProgress.get() + " remaining" + getServersInProgress(); + + throw new SocketTimeoutException(msg); } } else { waitUntilDone(Long.MAX_VALUE); } } catch (InterruptedException iex) { - throw new InterruptedIOException(iex.getMessage()); + long duration = EnvironmentEdgeManager.currentTime() - startTime; + String message = "Interrupted after waiting " + duration + "ms of " + operationTimeout + + "ms operation timeout, with " + actionsInProgress.get() + " remaining" + + getServersInProgress(); + if (!Strings.isNullOrEmpty(iex.getMessage())) { + message += ": " + iex.getMessage(); + } + throw new InterruptedIOException(message); } finally { if (callsInProgress != null) { for (CancellableRegionServerCallable clb : callsInProgress) { @@ -1299,6 +1323,29 @@ public void waitUntilDone() throws InterruptedIOException { } } + private String getServersInProgress() { + if (callsInProgress != null) { + Map serversInProgress = new HashMap<>(callsInProgress.size()); + for (CancellableRegionServerCallable callable : callsInProgress) { + if (callable instanceof MultiServerCallable) { + MultiServerCallable multiServerCallable = (MultiServerCallable) callable; + int numAttempt = multiServerCallable.getNumAttempt(); + serversInProgress.compute(multiServerCallable.getServerName(), + (k, v) -> v == null ? numAttempt : Math.max(v, numAttempt)); + } + } + + if (serversInProgress.size() > 0) { + return " on servers: " + serversInProgress.entrySet().stream() + .sorted(Comparator.> comparingInt(Map.Entry::getValue) + .reversed()) + .map(entry -> entry.getKey() + "(" + entry.getValue() + " attempts)") + .collect(Collectors.joining(", ")); + } + } + return ""; + } + private boolean waitUntilDone(long cutoff) throws InterruptedException { boolean hasWait = cutoff != Long.MAX_VALUE; long lastLog = EnvironmentEdgeManager.currentTime(); @@ -1365,10 +1412,10 @@ private ConnectionImplementation.ServerErrorTracker createServerErrorTracker() { * Create a callable. Isolated to be easily overridden in the tests. */ private MultiServerCallable createCallable(final ServerName server, TableName tableName, - final MultiAction multi) { + final MultiAction multi, int numAttempt) { return new MultiServerCallable(asyncProcess.connection, tableName, server, multi, asyncProcess.rpcFactory.newController(), rpcTimeout, tracker, multi.getPriority(), - requestAttributes); + requestAttributes, numAttempt); } private void updateResult(int index, Object result) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRpcRetryingCallerFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRpcRetryingCallerFactory.java index 1ea2a1ad7dd4..50d9d36a2b58 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRpcRetryingCallerFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRpcRetryingCallerFactory.java @@ -209,6 +209,8 @@ public class ScanSingleRegionCallerBuilder extends BuilderBase { private long rpcTimeoutNs; + private long lastNextCallNanos = System.nanoTime(); + private int priority = PRIORITY_UNSET; private Map requestAttributes = Collections.emptyMap(); @@ -275,6 +277,11 @@ public ScanSingleRegionCallerBuilder pause(long pause, TimeUnit unit) { return this; } + public ScanSingleRegionCallerBuilder lastNextCallNanos(long nanos) { + this.lastNextCallNanos = nanos; + return this; + } + public ScanSingleRegionCallerBuilder pauseForServerOverloaded(long pause, TimeUnit unit) { this.pauseNsForServerOverloaded = unit.toNanos(pause); return this; @@ -311,7 +318,7 @@ public AsyncScanSingleRegionRpcRetryingCaller build() { return new AsyncScanSingleRegionRpcRetryingCaller(retryTimer, conn, scan, scanMetrics, scannerId, resultCache, consumer, stub, loc, isRegionServerRemote, priority, scannerLeaseTimeoutPeriodNs, pauseNs, pauseNsForServerOverloaded, maxAttempts, - scanTimeoutNs, rpcTimeoutNs, startLogErrorsCnt, requestAttributes); + scanTimeoutNs, rpcTimeoutNs, lastNextCallNanos, startLogErrorsCnt, requestAttributes); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.java index 428a3c65507d..55e09a53ffe2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import static org.apache.hadoop.hbase.client.ConnectionUtils.incMillisBetweenNextsMetrics; import static org.apache.hadoop.hbase.client.ConnectionUtils.incRPCCallsMetrics; import static org.apache.hadoop.hbase.client.ConnectionUtils.incRPCRetriesMetrics; import static org.apache.hadoop.hbase.client.ConnectionUtils.noMoreResultsForReverseScan; @@ -120,7 +121,9 @@ class AsyncScanSingleRegionRpcRetryingCaller { private boolean includeNextStartRowWhenError; - private long nextCallStartNs; + private long lastNextCallNanos; + + private long nextCallStartNanos; private int tries; @@ -334,7 +337,7 @@ public AsyncScanSingleRegionRpcRetryingCaller(Timer retryTimer, AsyncConnectionI AdvancedScanResultConsumer consumer, Interface stub, HRegionLocation loc, boolean isRegionServerRemote, int priority, long scannerLeaseTimeoutPeriodNs, long pauseNs, long pauseNsForServerOverloaded, int maxAttempts, long scanTimeoutNs, long rpcTimeoutNs, - int startLogErrorsCnt, Map requestAttributes) { + long lastNextCallNanos, int startLogErrorsCnt, Map requestAttributes) { this.retryTimer = retryTimer; this.conn = conn; this.scan = scan; @@ -349,6 +352,7 @@ public AsyncScanSingleRegionRpcRetryingCaller(Timer retryTimer, AsyncConnectionI this.maxAttempts = maxAttempts; this.scanTimeoutNs = scanTimeoutNs; this.rpcTimeoutNs = rpcTimeoutNs; + this.lastNextCallNanos = lastNextCallNanos; this.startLogErrorsCnt = startLogErrorsCnt; if (scan.isReversed()) { completeWhenNoMoreResultsInRegion = this::completeReversedWhenNoMoreResultsInRegion; @@ -365,8 +369,12 @@ public AsyncScanSingleRegionRpcRetryingCaller(Timer retryTimer, AsyncConnectionI new HBaseServerExceptionPauseManager(pauseNs, pauseNsForServerOverloaded, scanTimeoutNs); } + public long getLastNextCallNanos() { + return lastNextCallNanos; + } + private long elapsedMs() { - return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - nextCallStartNs); + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - nextCallStartNanos); } private void closeScanner() { @@ -433,7 +441,7 @@ private void onError(Throwable error) { } OptionalLong maybePauseNsToUse = - pauseManager.getPauseNsFromException(error, tries, nextCallStartNs); + pauseManager.getPauseNsFromException(error, tries, nextCallStartNanos); if (!maybePauseNsToUse.isPresent()) { completeExceptionally(!scannerClosed); return; @@ -581,7 +589,7 @@ private void call() { // new one. long callTimeoutNs; if (scanTimeoutNs > 0) { - long remainingNs = scanTimeoutNs - (System.nanoTime() - nextCallStartNs); + long remainingNs = scanTimeoutNs - (System.nanoTime() - nextCallStartNanos); if (remainingNs <= 0) { completeExceptionally(true); return; @@ -609,7 +617,10 @@ private void next() { nextCallSeq++; tries = 1; exceptions.clear(); - nextCallStartNs = System.nanoTime(); + nextCallStartNanos = System.nanoTime(); + incMillisBetweenNextsMetrics(scanMetrics, + TimeUnit.NANOSECONDS.toMillis(nextCallStartNanos - lastNextCallNanos)); + lastNextCallNanos = nextCallStartNanos; call(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java index 29fb2c43bc0b..8737172deee5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java @@ -193,9 +193,7 @@ public void mutate(List ms) long toAddSize = 0; int toAddCount = 0; for (Mutation m : ms) { - if (m instanceof Put) { - ConnectionUtils.validatePut((Put) m, maxKeyValueSize); - } + ConnectionUtils.validateMutation(m, maxKeyValueSize); toAddSize += m.heapSize(); ++toAddCount; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 28b26ba648b2..ab6b25b340d4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -2062,6 +2062,12 @@ public HBaseProtos.LogEntry getLogEntries(RpcController controller, return stub.getLogEntries(controller, request); } + @Override + public MasterProtos.ReopenTableRegionsResponse reopenTableRegions(RpcController controller, + MasterProtos.ReopenTableRegionsRequest request) throws ServiceException { + return stub.reopenTableRegions(controller, request); + } + @Override public ModifyTableStoreFileTrackerResponse modifyTableStoreFileTracker( RpcController controller, ModifyTableStoreFileTrackerRequest request) @@ -2082,6 +2088,14 @@ public FlushMasterStoreResponse flushMasterStore(RpcController controller, return stub.flushMasterStore(controller, request); } + @Override + public MasterProtos.RestoreBackupSystemTableResponse restoreBackupSystemTable( + RpcController rpcController, + MasterProtos.RestoreBackupSystemTableRequest restoreBackupSystemTableRequest) + throws ServiceException { + return stub.restoreBackupSystemTable(rpcController, restoreBackupSystemTableRequest); + } + @Override public ReplicationPeerModificationSwitchResponse replicationPeerModificationSwitch( RpcController controller, ReplicationPeerModificationSwitchRequest request) @@ -2333,7 +2347,7 @@ public TableState getTableState(TableName tableName) throws IOException { @Override public RpcRetryingCallerFactory getNewRpcRetryingCallerFactory(Configuration conf) { return RpcRetryingCallerFactory.instantiate(conf, connectionConfig, this.interceptor, - this.getStatisticsTracker(), metrics); + this.stats, metrics); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java index 2beb107f2a88..7a3dd9761beb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java @@ -419,6 +419,13 @@ static void incRegionCountMetrics(ScanMetrics scanMetrics) { scanMetrics.addToCounter(ScanMetrics.REGIONS_SCANNED_METRIC_NAME, 1); } + static void incMillisBetweenNextsMetrics(ScanMetrics scanMetrics, long millis) { + if (scanMetrics == null) { + return; + } + scanMetrics.addToCounter(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME, millis); + } + /** * Connect the two futures, if the src future is done, then mark the dst future as done. And if * the dst future is done, then cancel the src future. This is used for timeline consistent read. @@ -506,13 +513,20 @@ static CompletableFuture timelineConsistentRead(AsyncRegionLocator locato return future; } - // validate for well-formedness - static void validatePut(Put put, int maxKeyValueSize) { - if (put.isEmpty()) { - throw new IllegalArgumentException("No columns to insert"); + // Validate individual Mutation + static void validateMutation(Mutation mutation, int maxKeyValueSize) { + // Skip Delete + if (mutation instanceof Delete) return; + + // 1. Check if empty + if (mutation.isEmpty()) { + throw new IllegalArgumentException( + "No columns to " + mutation.getClass().getSimpleName().toLowerCase()); } + + // 2. Check if size exceeds maxKeyValueSize if (maxKeyValueSize > 0) { - for (List list : put.getFamilyCellMap().values()) { + for (List list : mutation.getFamilyCellMap().values()) { for (Cell cell : list) { if (cell.getSerializedSize() > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); @@ -522,11 +536,10 @@ static void validatePut(Put put, int maxKeyValueSize) { } } - static void validatePutsInRowMutations(RowMutations rowMutations, int maxKeyValueSize) { + // Validate RowMutations + static void validateRowMutations(RowMutations rowMutations, int maxKeyValueSize) { for (Mutation mutation : rowMutations.getMutations()) { - if (mutation instanceof Put) { - validatePut((Put) mutation, maxKeyValueSize); - } + validateMutation(mutation, maxKeyValueSize); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CoprocessorDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CoprocessorDescriptorBuilder.java index cb0caca21b0e..00d666e38d82 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CoprocessorDescriptorBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CoprocessorDescriptorBuilder.java @@ -19,6 +19,7 @@ import java.util.Collections; import java.util.Map; +import java.util.NavigableMap; import java.util.Objects; import java.util.Optional; import java.util.TreeMap; @@ -42,7 +43,8 @@ public static CoprocessorDescriptorBuilder newBuilder(String className) { private final String className; private String jarPath; private int priority = Coprocessor.PRIORITY_USER; - private Map properties = new TreeMap(); + // HBASE-29706 Fix hashcode calculation. Use NavigableMap instead of Map + private NavigableMap properties = new TreeMap<>(); public CoprocessorDescriptorBuilder setJarPath(String jarPath) { this.jarPath = jarPath; @@ -76,10 +78,10 @@ private static final class CoprocessorDescriptorImpl implements CoprocessorDescr private final String className; private final String jarPath; private final int priority; - private final Map properties; + private final NavigableMap properties; private CoprocessorDescriptorImpl(String className, String jarPath, int priority, - Map properties) { + NavigableMap properties) { this.className = className; this.jarPath = jarPath; this.priority = priority; @@ -102,8 +104,8 @@ public int getPriority() { } @Override - public Map getProperties() { - return Collections.unmodifiableMap(properties); + public NavigableMap getProperties() { + return Collections.unmodifiableNavigableMap(properties); } @Override @@ -111,5 +113,28 @@ public String toString() { return "class:" + className + ", jarPath:" + jarPath + ", priority:" + priority + ", properties:" + properties; } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof CoprocessorDescriptor)) { + return false; + } + CoprocessorDescriptor other = (CoprocessorDescriptor) obj; + if ( + priority != other.getPriority() || !Objects.equals(className, other.getClassName()) + || !Objects.equals(getJarPath(), other.getJarPath()) + ) { + return false; + } + return Objects.equals(getProperties(), other.getProperties()); + } + + @Override + public int hashCode() { + return Objects.hash(className, jarPath, priority, properties); + } } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index d8b9c9dd9a68..be13415a6b59 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -213,6 +213,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesRequest; @@ -409,6 +411,34 @@ protected ModifyTableResponse rpcCall() throws Exception { return new ModifyTableFuture(this, td.getTableName(), response); } + @Override + public Future reopenTableRegionsAsync(TableName tableName) throws IOException { + return reopenTableRegionsAsync(tableName, Collections.emptyList()); + } + + @Override + public Future reopenTableRegionsAsync(TableName tableName, List regions) + throws IOException { + List regionNames = + regions.stream().map(RegionInfo::getRegionName).collect(Collectors.toList()); + + ReopenTableRegionsResponse response = executeCallable( + new MasterCallable(getConnection(), getRpcControllerFactory()) { + long nonceGroup = ng.getNonceGroup(); + long nonce = ng.newNonce(); + + @Override + protected ReopenTableRegionsResponse rpcCall() throws Exception { + setPriority(tableName); + ReopenTableRegionsRequest request = RequestConverter + .buildReopenTableRegionsRequest(tableName, regionNames, nonceGroup, nonce); + return master.reopenTableRegions(getRpcController(), request); + + } + }); + return new ReopenTableRegionsFuture(this, tableName, response); + } + @Override public Future modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT) throws IOException { @@ -2095,6 +2125,22 @@ public List getCachedFilesList(ServerName serverName) throws IOException this.connection.getAdmin(serverName)); } + @Override + public void restoreBackupSystemTable(String snapshotName) throws IOException { + long pid = + executeCallable(new MasterCallable(getConnection(), getRpcControllerFactory()) { + @Override + protected Long rpcCall() throws Exception { + return master.restoreBackupSystemTable(getRpcController(), + MasterProtos.RestoreBackupSystemTableRequest.newBuilder().setSnapshotName(snapshotName) + .build()) + .getProcId(); + } + }); + ProcedureFuture future = new ProcedureFuture<>(this, pid); + get(future, getProcedureTimeout, TimeUnit.MILLISECONDS); + } + private MasterCallable getTruncateRegionCallable(TableName tableName, RegionInfo hri) { return new MasterCallable(getConnection(), @@ -2125,6 +2171,19 @@ public String getOperationType() { } } + private static class ReopenTableRegionsFuture extends TableFuture { + public ReopenTableRegionsFuture(HBaseAdmin admin, TableName tableName, + ReopenTableRegionsResponse response) { + super(admin, tableName, + (response != null && response.hasProcId()) ? response.getProcId() : null); + } + + @Override + public String getOperationType() { + return "REOPEN_TABLE_REGIONS"; + } + } + private static class ModifyTableFuture extends TableFuture { public ModifyTableFuture(final HBaseAdmin admin, final TableName tableName, final ModifyTableResponse response) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java index 8767257bf5b7..31b38cf55dca 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -935,11 +935,20 @@ protected CheckAndMutateResult rpcCall() throws Exception { getLocation().getRegionInfo().getRegionName(), row, family, qualifier, op, value, filter, timeRange, mutation, nonceGroup, nonce, queryMetricsEnabled); MutateResponse response = doMutate(request); + CheckAndMutateResult result; if (response.hasResult()) { - return new CheckAndMutateResult(response.getProcessed(), + result = new CheckAndMutateResult(response.getProcessed(), ProtobufUtil.toResult(response.getResult(), getRpcControllerCellScanner())); + } else { + result = new CheckAndMutateResult(response.getProcessed(), null); } - return new CheckAndMutateResult(response.getProcessed(), null); + + if (response.hasMetrics()) { + QueryMetrics metrics = ProtobufUtil.toQueryMetrics(response.getMetrics()); + result.setMetrics(metrics); + } + + return result; } }; return rpcCallerFactory. newCaller(this.writeRpcTimeoutMs) @@ -1096,7 +1105,7 @@ public void close() throws IOException { // validate for well-formedness private void validatePut(final Put put) throws IllegalArgumentException { - ConnectionUtils.validatePut(put, connConfiguration.getMaxKeyValueSize()); + ConnectionUtils.validateMutation(put, connConfiguration.getMaxKeyValueSize()); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java index e81be5bcb253..562bc960f74a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java @@ -180,7 +180,7 @@ public boolean put(final TableName tableName, final Put put, int maxAttempts) { } try { - ConnectionUtils.validatePut(put, maxKeyValueSize); + ConnectionUtils.validateMutation(put, maxKeyValueSize); // Allow mocking to get at the connection, but don't expose the connection to users. ClusterConnection conn = (ClusterConnection) getConnection(); // AsyncProcess in the FlushWorker should take care of refreshing the location cache diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java index 6ba0832b26e5..33933dd5684f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java @@ -48,14 +48,17 @@ @InterfaceAudience.Private class MultiServerCallable extends CancellableRegionServerCallable { private MultiAction multiAction; + private final int numAttempt; private boolean cellBlock; MultiServerCallable(final ClusterConnection connection, final TableName tableName, final ServerName location, final MultiAction multi, RpcController rpcController, int rpcTimeout, - RetryingTimeTracker tracker, int priority, Map requestAttributes) { + RetryingTimeTracker tracker, int priority, Map requestAttributes, + int numAttempt) { super(connection, tableName, null, rpcController, rpcTimeout, tracker, priority, requestAttributes); this.multiAction = multi; + this.numAttempt = numAttempt; // RegionServerCallable has HRegionLocation field, but this is a multi-region request. // Using region info from parent HRegionLocation would be a mistake for this class; so // we will store the server here, and throw if someone tries to obtain location/regioninfo. @@ -63,6 +66,10 @@ class MultiServerCallable extends CancellableRegionServerCallable this.cellBlock = isCellBlock(); } + public int getNumAttempt() { + return numAttempt; + } + public void reset(ServerName location, MultiAction multiAction) { this.location = new HRegionLocation(null, location); this.multiAction = multiAction; diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell2.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpOperationInterceptor.java similarity index 57% rename from hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell2.java rename to hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpOperationInterceptor.java index 0bf67cdc8b45..c8a3d7d6d9f4 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell2.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpOperationInterceptor.java @@ -17,21 +17,32 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.apache.yetus.audience.InterfaceAudience; -@Category({ ClientTests.class, LargeTests.class }) -public class TestAdminShell2 extends AbstractTestShell { +/** + * No-op implementation of OperationInterceptor that does nothing. Used as the default when no + * custom interceptor is configured. + */ +@InterfaceAudience.Private +class NoOpOperationInterceptor extends OperationInterceptor { + + NoOpOperationInterceptor() { + super(); + } + + @Override + public void beforeAttempt(RetryingCallable callable) { + } - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAdminShell2.class); + @Override + public void afterAttemptSuccess(RetryingCallable callable, Object result) { + } + + @Override + public void afterAttemptFailure(RetryingCallable callable, Throwable cause) { + } @Override - protected String getIncludeList() { - return "admin2_test.rb"; + public void afterOperation() { } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptor.java new file mode 100644 index 000000000000..ddb78554df54 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptor.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Modern per-operation interceptor for HBase client operations. Each operation gets a fresh + * interceptor instance, eliminating thread-safety concerns and making implementation simple. + *

+ * This interceptor tracks both single operations (get, put, delete) and batch operations (multi-row + * operations). Batch operations are treated as single operations for simplicity. + *

+ * All fields are automatically populated by the HBase client. + *

+ * Usage example: + * + * 

+ * public class MyInterceptor extends OperationInterceptor {
+ *   public MyInterceptor(long operationStartTime) {
+ *     super(operationStartTime);
+ *   }
+ *
+ *   public void afterAttemptFailure(RetryingCallable callable, Throwable cause) {
+ *     long attemptDuration = System.currentTimeMillis() - getCurrentAttemptStartTime();
+ *     long operationDuration = System.currentTimeMillis() - getOperationStartTime();
+ *
+ *     // Fast-fail after 5 attempts or 30 seconds
+ *     if (getAttemptNumber() >= 4 || operationDuration > 30000) {
+ *       throw new FastFailException("Operation taking too long");
+ *     }
+ *
+ *     recordMetric("attempt.failure.duration", attemptDuration);
+ *     recordMetric("attempt.failure.type", cause.getClass().getSimpleName());
+ *   }
+ * }
+ *
+ */ +@InterfaceAudience.Public +public abstract class OperationInterceptor { + + /** + * Called before each attempt. + * @param callable the callable about to be executed + * @throws IOException the implementer may throw if they find issue with the request + */ + public abstract void beforeAttempt(RetryingCallable callable) throws IOException; + + /** + * Called after successful attempt completion. + * @param callable the callable that was executed + * @param result the result returned by the attempt (may be null) + * @throws IOException the implementer may throw if they find issue with the result + */ + public abstract void afterAttemptSuccess(RetryingCallable callable, Object result) + throws IOException; + + /** + * Called after attempt failure, before retry logic. + * @param callable the callable that failed + * @param cause the exception that caused the failure + * @throws IOException the implementer may throw an alternative exception + */ + public abstract void afterAttemptFailure(RetryingCallable callable, Throwable cause) + throws IOException; + + /** + * Called at the end of the operation, regardless of success or failure. This is guaranteed to be + * called exactly once per operation. + */ + public abstract void afterOperation(); +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptorFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptorFactory.java new file mode 100644 index 000000000000..88e15a87b17d --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptorFactory.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Factory for creating per-operation interceptors. Each HBase client operation (get, put, batch, + * etc.) will call createInterceptor() to get a fresh interceptor instance, ensuring thread safety + * and simplicity. + *

+ * Implementations should be lightweight and thread-safe, as the factory itself may be shared across + * multiple threads. However, the interceptors created by the factory are used by only one operation + * at a time. + *

+ * Configuration is injected once during factory construction and can be used to customize + * interceptor behavior. + */ +@InterfaceAudience.Public +public interface OperationInterceptorFactory { + + /** + * Configuration key for specifying the OperationInterceptorFactory implementation. The specified + * class must implement OperationInterceptorFactory and have a no-argument constructor. + */ + String HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL = "hbase.client.operation.interceptor.impl"; + + /** + * Create a new interceptor instance for a single operation. This method may be called + * concurrently from multiple threads. + * @return a new OperationInterceptor instance, never null + */ + OperationInterceptor createInterceptor(); + + /** + * A no-op factory that creates interceptors that do nothing. Used as the default when no custom + * interceptor is configured. + */ + OperationInterceptorFactory NO_OP = new OperationInterceptorFactory() { + @Override + public OperationInterceptor createInterceptor() { + return new NoOpOperationInterceptor(); + } + }; +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index bd01b2d247b1..ac88fb0c9747 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -256,6 +256,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RunCatalogScanRequest; @@ -674,6 +676,22 @@ public CompletableFuture modifyTable(TableDescriptor desc, boolean reopenR new ModifyTableProcedureBiConsumer(this, desc.getTableName())); } + @Override + public CompletableFuture reopenTableRegions(TableName tableName) { + return reopenTableRegions(tableName, Collections.emptyList()); + } + + @Override + public CompletableFuture reopenTableRegions(TableName tableName, List regions) { + List regionNames = + regions.stream().map(RegionInfo::getRegionName).collect(Collectors.toList()); + return this. procedureCall(tableName, + RequestConverter.buildReopenTableRegionsRequest(tableName, regionNames, ng.getNonceGroup(), + ng.newNonce()), + (s, c, req, done) -> s.reopenTableRegions(c, req, done), (resp) -> resp.getProcId(), + new ReopenTableRegionsProcedureBiConsumer(this, tableName)); + } + @Override public CompletableFuture modifyTableStoreFileTracker(TableName tableName, String dstSFT) { return this. compact(TableName tableName, byte[] columnFamily } if (locations == null || locations.isEmpty()) { future.completeExceptionally(new TableNotFoundException(tableName)); + return; } CompletableFuture[] compactFutures = locations.stream().filter(l -> l.getRegion() != null) @@ -2716,6 +2735,19 @@ void onError(Throwable error) { } } + private static class RestoreBackupSystemTableProcedureBiConsumer extends ProcedureBiConsumer { + + @Override + void onFinished() { + LOG.info("RestoreBackupSystemTableProcedure completed"); + } + + @Override + void onError(Throwable error) { + LOG.info("RestoreBackupSystemTableProcedure failed with {}", error.getMessage()); + } + } + private static class CreateTableProcedureBiConsumer extends TableProcedureBiConsumer { CreateTableProcedureBiConsumer(TableName tableName) { @@ -2740,6 +2772,18 @@ String getOperationType() { } } + private static class ReopenTableRegionsProcedureBiConsumer extends TableProcedureBiConsumer { + + ReopenTableRegionsProcedureBiConsumer(AsyncAdmin admin, TableName tableName) { + super(tableName); + } + + @Override + String getOperationType() { + return "REOPEN_TABLE_REGIONS"; + } + } + private static class ModifyTableStoreFileTrackerProcedureBiConsumer extends TableProcedureBiConsumer { @@ -2997,10 +3041,16 @@ GetProcedureResultResponse> call(controller, stub, .call(), (response, error) -> { if (error != null) { - LOG.warn("failed to get the procedure result procId={}", procId, - ConnectionUtils.translateException(error)); - retryTimer.newTimeout(t -> getProcedureResult(procId, future, retries + 1), - ConnectionUtils.getPauseTime(pauseNs, retries), TimeUnit.NANOSECONDS); + Throwable exc = ConnectionUtils.translateException(error); + if (exc instanceof DoNotRetryIOException) { + // stop retrying on DNRIOE + future.completeExceptionally(exc); + } else { + LOG.warn("failed to get the procedure result procId={}", procId, exc); + retryTimer.newTimeout(t -> getProcedureResult(procId, future, retries + 1), + ConnectionUtils.getPauseTime(pauseNs, retries), TimeUnit.NANOSECONDS); + + } return; } if (response.getState() == GetProcedureResultResponse.State.RUNNING) { @@ -4299,4 +4349,16 @@ List> adminCall(controller, stub, request.build(), resp -> resp.getCachedFilesList())) .serverName(serverName).call(); } + + @Override + public CompletableFuture restoreBackupSystemTable(String snapshotName) { + MasterProtos.RestoreBackupSystemTableRequest request = + MasterProtos.RestoreBackupSystemTableRequest.newBuilder().setSnapshotName(snapshotName) + .build(); + return this. procedureCall(request, + MasterService.Interface::restoreBackupSystemTable, + MasterProtos.RestoreBackupSystemTableResponse::getProcId, + new RestoreBackupSystemTableProcedureBiConsumer()); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncTableImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncTableImpl.java index 05dd5ea6394b..05cc3e5ed00e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncTableImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncTableImpl.java @@ -18,11 +18,10 @@ package org.apache.hadoop.hbase.client; import static java.util.stream.Collectors.toList; -import static org.apache.hadoop.hbase.client.ConnectionUtils.checkHasFamilies; import static org.apache.hadoop.hbase.client.ConnectionUtils.isEmptyStopRow; import static org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead; -import static org.apache.hadoop.hbase.client.ConnectionUtils.validatePut; -import static org.apache.hadoop.hbase.client.ConnectionUtils.validatePutsInRowMutations; +import static org.apache.hadoop.hbase.client.ConnectionUtils.validateMutation; +import static org.apache.hadoop.hbase.client.ConnectionUtils.validateRowMutations; import static org.apache.hadoop.hbase.trace.TraceUtil.tracedFuture; import static org.apache.hadoop.hbase.trace.TraceUtil.tracedFutures; import static org.apache.hadoop.hbase.util.FutureUtils.addListener; @@ -287,7 +286,7 @@ public CompletableFuture get(Get get) { @Override public CompletableFuture put(Put put) { - validatePut(put, conn.connConf.getMaxKeyValueSize()); + validateMutation(put, conn.connConf.getMaxKeyValueSize()); final Supplier supplier = newTableOperationSpanBuilder().setOperation(put); return tracedFuture(() -> this. newCaller(put, writeRpcTimeoutNs) .action((controller, loc, stub) -> RawAsyncTableImpl. voidMutate(controller, loc, stub, @@ -306,7 +305,7 @@ public CompletableFuture delete(Delete delete) { @Override public CompletableFuture append(Append append) { - checkHasFamilies(append); + validateMutation(append, conn.connConf.getMaxKeyValueSize()); final Supplier supplier = newTableOperationSpanBuilder().setOperation(append); return tracedFuture(() -> { long nonceGroup = conn.getNonceGenerator().getNonceGroup(); @@ -321,7 +320,7 @@ public CompletableFuture append(Append append) { @Override public CompletableFuture increment(Increment increment) { - checkHasFamilies(increment); + validateMutation(increment, conn.connConf.getMaxKeyValueSize()); final Supplier supplier = newTableOperationSpanBuilder().setOperation(increment); return tracedFuture(() -> { long nonceGroup = conn.getNonceGenerator().getNonceGroup(); @@ -387,7 +386,7 @@ private void preCheck() { @Override public CompletableFuture thenPut(Put put) { - validatePut(put, conn.connConf.getMaxKeyValueSize()); + validateMutation(put, conn.connConf.getMaxKeyValueSize()); preCheck(); final Supplier supplier = newTableOperationSpanBuilder() .setOperation(HBaseSemanticAttributes.Operation.CHECK_AND_MUTATE) @@ -421,7 +420,7 @@ public CompletableFuture thenDelete(Delete delete) { @Override public CompletableFuture thenMutate(RowMutations mutations) { preCheck(); - validatePutsInRowMutations(mutations, conn.connConf.getMaxKeyValueSize()); + validateRowMutations(mutations, conn.connConf.getMaxKeyValueSize()); final Supplier supplier = newTableOperationSpanBuilder() .setOperation(HBaseSemanticAttributes.Operation.CHECK_AND_MUTATE) .setContainerOperations(mutations); @@ -463,7 +462,7 @@ public CheckAndMutateWithFilterBuilder timeRange(TimeRange timeRange) { @Override public CompletableFuture thenPut(Put put) { - validatePut(put, conn.connConf.getMaxKeyValueSize()); + validateMutation(put, conn.connConf.getMaxKeyValueSize()); final Supplier supplier = newTableOperationSpanBuilder() .setOperation(HBaseSemanticAttributes.Operation.CHECK_AND_MUTATE) .setContainerOperations(put); @@ -494,7 +493,7 @@ public CompletableFuture thenDelete(Delete delete) { @Override public CompletableFuture thenMutate(RowMutations mutations) { - validatePutsInRowMutations(mutations, conn.connConf.getMaxKeyValueSize()); + validateRowMutations(mutations, conn.connConf.getMaxKeyValueSize()); final Supplier supplier = newTableOperationSpanBuilder() .setOperation(HBaseSemanticAttributes.Operation.CHECK_AND_MUTATE) .setContainerOperations(mutations); @@ -525,9 +524,7 @@ public CompletableFuture checkAndMutate(CheckAndMutate che || checkAndMutate.getAction() instanceof Append ) { Mutation mutation = (Mutation) checkAndMutate.getAction(); - if (mutation instanceof Put) { - validatePut((Put) mutation, conn.connConf.getMaxKeyValueSize()); - } + validateMutation(mutation, conn.connConf.getMaxKeyValueSize()); long nonceGroup = conn.getNonceGenerator().getNonceGroup(); long nonce = conn.getNonceGenerator().newNonce(); return RawAsyncTableImpl.this @@ -544,7 +541,7 @@ public CompletableFuture checkAndMutate(CheckAndMutate che .call(); } else if (checkAndMutate.getAction() instanceof RowMutations) { RowMutations rowMutations = (RowMutations) checkAndMutate.getAction(); - validatePutsInRowMutations(rowMutations, conn.connConf.getMaxKeyValueSize()); + validateRowMutations(rowMutations, conn.connConf.getMaxKeyValueSize()); long nonceGroup = conn.getNonceGenerator().getNonceGroup(); long nonce = conn.getNonceGenerator().newNonce(); return RawAsyncTableImpl.this @@ -623,7 +620,7 @@ public void run(MultiResponse resp) { @Override public CompletableFuture mutateRow(RowMutations mutations) { - validatePutsInRowMutations(mutations, conn.connConf.getMaxKeyValueSize()); + validateRowMutations(mutations, conn.connConf.getMaxKeyValueSize()); long nonceGroup = conn.getNonceGenerator().getNonceGroup(); long nonce = conn.getNonceGenerator().newNonce(); final Supplier supplier = @@ -732,18 +729,19 @@ private List> voidMutate(List actions) { private List> batch(List actions, long rpcTimeoutNs) { for (Row action : actions) { - if (action instanceof Put) { - validatePut((Put) action, conn.connConf.getMaxKeyValueSize()); + if (action instanceof Mutation) { + validateMutation((Mutation) action, conn.connConf.getMaxKeyValueSize()); } else if (action instanceof CheckAndMutate) { CheckAndMutate checkAndMutate = (CheckAndMutate) action; - if (checkAndMutate.getAction() instanceof Put) { - validatePut((Put) checkAndMutate.getAction(), conn.connConf.getMaxKeyValueSize()); + if (checkAndMutate.getAction() instanceof Mutation) { + validateMutation((Mutation) checkAndMutate.getAction(), + conn.connConf.getMaxKeyValueSize()); } else if (checkAndMutate.getAction() instanceof RowMutations) { - validatePutsInRowMutations((RowMutations) checkAndMutate.getAction(), + validateRowMutations((RowMutations) checkAndMutate.getAction(), conn.connConf.getMaxKeyValueSize()); } } else if (action instanceof RowMutations) { - validatePutsInRowMutations((RowMutations) action, conn.connConf.getMaxKeyValueSize()); + validateRowMutations((RowMutations) action, conn.connConf.getMaxKeyValueSize()); } } return conn.callerFactory.batch().table(tableName).actions(actions) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorRpcChannelImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorRpcChannelImpl.java index 8acadafbb068..16d2bbb726c0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorRpcChannelImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorRpcChannelImpl.java @@ -29,13 +29,14 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest; @@ -46,6 +47,8 @@ @InterfaceAudience.Private class RegionCoprocessorRpcChannelImpl implements RpcChannel { + private static final Logger LOG = LoggerFactory.getLogger(RegionCoprocessorRpcChannelImpl.class); + private final AsyncConnectionImpl conn; private final TableName tableName; @@ -76,10 +79,8 @@ private CompletableFuture rpcCall(MethodDescriptor method, Message requ if ( region != null && !Bytes.equals(loc.getRegionInfo().getRegionName(), region.getRegionName()) ) { - future.completeExceptionally(new DoNotRetryIOException( - "Region name is changed, expected " + region.getRegionNameAsString() + ", actual " - + loc.getRegionInfo().getRegionNameAsString())); - return future; + LOG.warn("Region name is changed, expected {}, actual {}", region.getRegionNameAsString(), + loc.getRegionInfo().getRegionNameAsString()); } CoprocessorServiceRequest csr = CoprocessorRpcUtils.getCoprocessorServiceRequest(method, request, row, loc.getRegionInfo().getRegionName()); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java index c062ad43e253..d913166b12c8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java @@ -21,6 +21,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Factory to create an {@link RpcRetryingCaller} @@ -28,24 +30,35 @@ @InterfaceAudience.Private public class RpcRetryingCallerFactory { + private static final Logger LOG = LoggerFactory.getLogger(RpcRetryingCallerFactory.class); + /** Configuration key for a custom {@link RpcRetryingCaller} */ public static final String CUSTOM_CALLER_CONF_KEY = "hbase.rpc.callerfactory.class"; private final ConnectionConfiguration connectionConf; private final RetryingCallerInterceptor interceptor; private final int startLogErrorsCnt; private final MetricsConnection metrics; + private final OperationInterceptorFactory operationInterceptorFactory; public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf) { - this(conf, connectionConf, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null); + this(conf, connectionConf, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null, + createOperationInterceptorFactory(conf)); } public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf, RetryingCallerInterceptor interceptor, MetricsConnection metrics) { + this(conf, connectionConf, interceptor, metrics, createOperationInterceptorFactory(conf)); + } + + public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf, + RetryingCallerInterceptor interceptor, MetricsConnection metrics, + OperationInterceptorFactory operationInterceptorFactory) { this.connectionConf = connectionConf; startLogErrorsCnt = conf.getInt(AsyncProcess.START_LOG_ERRORS_AFTER_COUNT_KEY, AsyncProcess.DEFAULT_START_LOG_ERRORS_AFTER_COUNT); this.interceptor = interceptor; this.metrics = metrics; + this.operationInterceptorFactory = operationInterceptorFactory; } /** @@ -56,7 +69,7 @@ public RpcRetryingCaller newCaller(int rpcTimeout) { // is cheap as it does not require parsing a complex structure. return new RpcRetryingCallerImpl<>(connectionConf.getPauseMillis(), connectionConf.getPauseMillisForServerOverloaded(), connectionConf.getRetriesNumber(), - interceptor, startLogErrorsCnt, rpcTimeout, metrics); + interceptor, startLogErrorsCnt, rpcTimeout, metrics, operationInterceptorFactory); } /** @@ -67,7 +80,8 @@ public RpcRetryingCaller newCaller() { // is cheap as it does not require parsing a complex structure. return new RpcRetryingCallerImpl<>(connectionConf.getPauseMillis(), connectionConf.getPauseMillisForServerOverloaded(), connectionConf.getRetriesNumber(), - interceptor, startLogErrorsCnt, connectionConf.getRpcTimeout(), metrics); + interceptor, startLogErrorsCnt, connectionConf.getRpcTimeout(), metrics, + operationInterceptorFactory); } @RestrictedApi(explanation = "Should only be called on process initialization", link = "", @@ -80,25 +94,36 @@ public static RpcRetryingCallerFactory instantiate(Configuration configuration, public static RpcRetryingCallerFactory instantiate(Configuration configuration, ConnectionConfiguration connectionConf, MetricsConnection metrics) { return instantiate(configuration, connectionConf, - RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null, metrics); + RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null, metrics, + createOperationInterceptorFactory(configuration)); } public static RpcRetryingCallerFactory instantiate(Configuration configuration, ConnectionConfiguration connectionConf, ServerStatisticTracker stats, MetricsConnection metrics) { return instantiate(configuration, connectionConf, - RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, stats, metrics); + RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, stats, metrics, + createOperationInterceptorFactory(configuration)); } public static RpcRetryingCallerFactory instantiate(Configuration configuration, ConnectionConfiguration connectionConf, RetryingCallerInterceptor interceptor, ServerStatisticTracker stats, MetricsConnection metrics) { + return instantiate(configuration, connectionConf, interceptor, stats, metrics, + createOperationInterceptorFactory(configuration)); + } + + public static RpcRetryingCallerFactory instantiate(Configuration configuration, + ConnectionConfiguration connectionConf, RetryingCallerInterceptor interceptor, + ServerStatisticTracker stats, MetricsConnection metrics, + OperationInterceptorFactory operationInterceptorFactory) { String clazzName = RpcRetryingCallerFactory.class.getName(); String rpcCallerFactoryClazz = configuration.get(RpcRetryingCallerFactory.CUSTOM_CALLER_CONF_KEY, clazzName); RpcRetryingCallerFactory factory; if (rpcCallerFactoryClazz.equals(clazzName)) { - factory = new RpcRetryingCallerFactory(configuration, connectionConf, interceptor, metrics); + factory = new RpcRetryingCallerFactory(configuration, connectionConf, interceptor, metrics, + operationInterceptorFactory); } else { factory = ReflectionUtils.instantiateWithCustomCtor(rpcCallerFactoryClazz, new Class[] { Configuration.class, ConnectionConfiguration.class }, @@ -106,4 +131,20 @@ public static RpcRetryingCallerFactory instantiate(Configuration configuration, } return factory; } + + private static OperationInterceptorFactory createOperationInterceptorFactory(Configuration conf) { + String clazz = conf.get(OperationInterceptorFactory.HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL); + if (clazz == null || clazz.isEmpty()) { + return OperationInterceptorFactory.NO_OP; + } + try { + Class factoryClass = + conf.getClassByName(clazz).asSubclass(OperationInterceptorFactory.class); + return ReflectionUtils.newInstance(factoryClass, conf); + } catch (ClassNotFoundException e) { + LOG.warn("Failed to load OperationInterceptorFactory class: {}, using NO_OP instead", clazz, + e); + return OperationInterceptorFactory.NO_OP; + } + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java index c692a2757d6f..6c73a391ced3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java @@ -66,10 +66,18 @@ public class RpcRetryingCallerImpl implements RpcRetryingCaller { private final RetryingCallerInterceptorContext context; private final RetryingTimeTracker tracker; private final MetricsConnection metrics; + private final OperationInterceptorFactory operationInterceptorFactory; public RpcRetryingCallerImpl(long pause, long pauseForServerOverloaded, int retries, RetryingCallerInterceptor interceptor, int startLogErrorsCnt, int rpcTimeout, MetricsConnection metricsConnection) { + this(pause, pauseForServerOverloaded, retries, interceptor, startLogErrorsCnt, rpcTimeout, + metricsConnection, OperationInterceptorFactory.NO_OP); + } + + public RpcRetryingCallerImpl(long pause, long pauseForServerOverloaded, int retries, + RetryingCallerInterceptor interceptor, int startLogErrorsCnt, int rpcTimeout, + MetricsConnection metricsConnection, OperationInterceptorFactory operationInterceptorFactory) { this.pause = pause; this.pauseForServerOverloaded = pauseForServerOverloaded; this.maxAttempts = retries2Attempts(retries); @@ -79,6 +87,7 @@ public RpcRetryingCallerImpl(long pause, long pauseForServerOverloaded, int retr this.tracker = new RetryingTimeTracker(); this.rpcTimeout = rpcTimeout; this.metrics = metricsConnection; + this.operationInterceptorFactory = operationInterceptorFactory; } @Override @@ -95,16 +104,25 @@ public T callWithRetries(RetryingCallable callable, int callTimeout) List exceptions = new ArrayList<>(); tracker.start(); context.clear(); + + OperationInterceptor operationInterceptor = operationInterceptorFactory.createInterceptor(); + for (int tries = 0;; tries++) { long expectedSleep; + try { // bad cache entries are cleared in the call to RetryingCallable#throwable() in catch block callable.prepare(tries != 0); interceptor.intercept(context.prepare(callable, tries)); - return callable.call(getTimeout(callTimeout)); + operationInterceptor.beforeAttempt(callable); + T result = callable.call(getTimeout(callTimeout)); + operationInterceptor.afterAttemptSuccess(callable, result); + operationInterceptor.afterOperation(); + return result; } catch (PreemptiveFastFailException e) { throw e; } catch (Throwable t) { + operationInterceptor.afterAttemptFailure(callable, t); ExceptionUtil.rethrowIfInterrupt(t); Throwable cause = t.getCause(); if (cause instanceof DoNotRetryIOException) { @@ -139,6 +157,7 @@ public T callWithRetries(RetryingCallable callable, int callTimeout) EnvironmentEdgeManager.currentTime(), toString()); exceptions.add(qt); if (tries >= maxAttempts - 1) { + operationInterceptor.afterOperation(); throw new RetriesExhaustedException(tries, exceptions); } @@ -201,10 +220,19 @@ private long singleCallDuration(final long expectedSleep) { public T callWithoutRetries(RetryingCallable callable, int callTimeout) throws IOException, RuntimeException { // The code of this method should be shared with withRetries. + long startTime = EnvironmentEdgeManager.currentTime(); + OperationInterceptor operationInterceptor = operationInterceptorFactory.createInterceptor(); + try { callable.prepare(false); - return callable.call(callTimeout); + operationInterceptor.beforeAttempt(callable); + T result = callable.call(callTimeout); + operationInterceptor.afterAttemptSuccess(callable, result); + operationInterceptor.afterOperation(); + return result; } catch (Throwable t) { + operationInterceptor.afterAttemptFailure(callable, t); + operationInterceptor.afterOperation(); Throwable t2 = translateException(t); ExceptionUtil.rethrowIfInterrupt(t2); // It would be nice to clear the location cache here. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java index 57f3cba86053..f38ba0c9f224 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java @@ -338,6 +338,12 @@ public ModifyTableResponse modifyTable(RpcController controller, ModifyTableRequ return stub.modifyTable(controller, request); } + @Override + public MasterProtos.ReopenTableRegionsResponse reopenTableRegions(RpcController controller, + MasterProtos.ReopenTableRegionsRequest request) throws ServiceException { + return stub.reopenTableRegions(controller, request); + } + @Override public ModifyNamespaceResponse modifyNamespace(RpcController controller, ModifyNamespaceRequest request) throws ServiceException { @@ -796,6 +802,13 @@ public FlushMasterStoreResponse flushMasterStore(RpcController controller, return stub.flushMasterStore(controller, request); } + @Override + public MasterProtos.RestoreBackupSystemTableResponse restoreBackupSystemTable( + RpcController controller, MasterProtos.RestoreBackupSystemTableRequest request) + throws ServiceException { + return stub.restoreBackupSystemTable(controller, request); + } + @Override public ReplicationPeerModificationSwitchResponse replicationPeerModificationSwitch( RpcController controller, ReplicationPeerModificationSwitchRequest request) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java index c017387d67fc..dfeadb972296 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; @@ -26,10 +27,16 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.zip.CRC32; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; /** * TableDescriptor contains the details about an HBase table such as the descriptors of all the @@ -337,4 +344,23 @@ default boolean matchReplicationScope(boolean enabled) { * {@link org.apache.hadoop.hbase.rsgroup.RSGroupInfo#DEFAULT_GROUP}. */ Optional getRegionServerGroup(); + + /** + * Computes a CRC32 hash of the table descriptor's protobuf representation. This hash can be used + * to detect changes in the table descriptor configuration. + * @return A hex string representation of the CRC32 hash, or "UNKNOWN" if computation fails + */ + default String getDescriptorHash() { + try { + HBaseProtos.TableSchema tableSchema = ProtobufUtil.toTableSchema(this); + ByteBuffer byteBuffer = ByteBuffer.wrap(tableSchema.toByteArray()); + CRC32 crc32 = new CRC32(); + crc32.update(byteBuffer); + return Long.toHexString(crc32.getValue()); + } catch (Exception e) { + Logger log = LoggerFactory.getLogger(TableDescriptor.class); + log.error("Failed to compute table descriptor hash for table {}", getTableName(), e); + return "UNKNOWN"; + } + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java index 3c8b7ad34b71..2ec5202f7fb5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java @@ -633,6 +633,11 @@ public static class ModifyableTableDescriptor private final Map families = new TreeMap<>(Bytes.BYTES_RAWCOMPARATOR); + /** + * Cached hash of the table descriptor. Computed lazily on first access. + */ + private volatile String descriptorHash; + /** * Construct a table descriptor specifying a TableName object * @param name Table name. TODO: make this private after removing the HTableDescriptor @@ -1619,6 +1624,14 @@ public Optional getRegionServerGroup() { return Optional.empty(); } } + + @Override + public String getDescriptorHash() { + if (descriptorHash == null) { + descriptorHash = TableDescriptor.super.getDescriptorHash(); + } + return descriptorHash; + } } private static Optional toCoprocessorDescriptor(String spec) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java index 916451df75da..eeaea716a443 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java @@ -52,6 +52,12 @@ public void moveToNextRegion() { currentRegionScanMetricsData.createCounter(COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME); currentRegionScanMetricsData.createCounter(BLOCK_BYTES_SCANNED_KEY_METRIC_NAME); currentRegionScanMetricsData.createCounter(FS_READ_TIME_METRIC_NAME); + currentRegionScanMetricsData.createCounter(BYTES_READ_FROM_FS_METRIC_NAME); + currentRegionScanMetricsData.createCounter(BYTES_READ_FROM_BLOCK_CACHE_METRIC_NAME); + currentRegionScanMetricsData.createCounter(BYTES_READ_FROM_MEMSTORE_METRIC_NAME); + currentRegionScanMetricsData.createCounter(BLOCK_READ_OPS_COUNT_METRIC_NAME); + currentRegionScanMetricsData.createCounter(RPC_SCAN_PROCESSING_TIME_METRIC_NAME); + currentRegionScanMetricsData.createCounter(RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME); } /** @@ -68,6 +74,13 @@ protected AtomicLong createCounter(String counterName) { public static final String BLOCK_BYTES_SCANNED_KEY_METRIC_NAME = "BLOCK_BYTES_SCANNED"; public static final String FS_READ_TIME_METRIC_NAME = "FS_READ_TIME"; + public static final String BYTES_READ_FROM_FS_METRIC_NAME = "BYTES_READ_FROM_FS"; + public static final String BYTES_READ_FROM_BLOCK_CACHE_METRIC_NAME = + "BYTES_READ_FROM_BLOCK_CACHE"; + public static final String BYTES_READ_FROM_MEMSTORE_METRIC_NAME = "BYTES_READ_FROM_MEMSTORE"; + public static final String BLOCK_READ_OPS_COUNT_METRIC_NAME = "BLOCK_READ_OPS_COUNT"; + public static final String RPC_SCAN_PROCESSING_TIME_METRIC_NAME = "RPC_SCAN_PROCESSING_TIME"; + public static final String RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME = "RPC_SCAN_QUEUE_WAIT_TIME"; /** * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0 @@ -102,6 +115,22 @@ protected AtomicLong createCounter(String counterName) { public final AtomicLong fsReadTime = createCounter(FS_READ_TIME_METRIC_NAME); + public final AtomicLong bytesReadFromFs = createCounter(BYTES_READ_FROM_FS_METRIC_NAME); + + public final AtomicLong bytesReadFromBlockCache = + createCounter(BYTES_READ_FROM_BLOCK_CACHE_METRIC_NAME); + + public final AtomicLong bytesReadFromMemstore = + createCounter(BYTES_READ_FROM_MEMSTORE_METRIC_NAME); + + public final AtomicLong blockReadOpsCount = createCounter(BLOCK_READ_OPS_COUNT_METRIC_NAME); + + public final AtomicLong rpcScanProcessingTime = + createCounter(RPC_SCAN_PROCESSING_TIME_METRIC_NAME); + + public final AtomicLong rpcScanQueueWaitTime = + createCounter(RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME); + /** * Sets counter with counterName to passed in value, does nothing if counter does not exist. If * region level scan metrics are enabled then sets the value of counter for the current region diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java index 3742eb8118a1..60f4b7932bfc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.provider.SaslClientAuthenticationProviders; import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.PoolMap; @@ -116,6 +117,8 @@ public abstract class AbstractRpcClient implements RpcC protected final UserProvider userProvider; protected final CellBlockBuilder cellBlockBuilder; + protected final SaslClientAuthenticationProviders providers; + protected final int minIdleTimeBeforeClose; // if the connection is idle for more than this // time (in ms), it will be closed at any moment. protected final int maxRetries; // the max. no. of retries for socket connections @@ -184,6 +187,8 @@ public AbstractRpcClient(Configuration conf, String clusterId, SocketAddress loc conf.getInt(HConstants.HBASE_CLIENT_PERSERVER_REQUESTS_THRESHOLD, HConstants.DEFAULT_HBASE_CLIENT_PERSERVER_REQUESTS_THRESHOLD); + this.providers = new SaslClientAuthenticationProviders(conf); + this.connections = new PoolMap<>(getPoolType(conf), getPoolSize(conf)); this.cleanupIdleConnectionTask = IDLE_CONN_SWEEPER.scheduleAtFixedRate(new Runnable() { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java index fe46c97a6539..c52a00cd81e3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java @@ -220,8 +220,9 @@ public void cleanup(IOException e) { BlockingRpcConnection(BlockingRpcClient rpcClient, ConnectionId remoteId) throws IOException { super(rpcClient.conf, AbstractRpcClient.WHEEL_TIMER, remoteId, rpcClient.clusterId, - rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.codec, rpcClient.compressor, - rpcClient.cellBlockBuilder, rpcClient.metrics, rpcClient.connectionAttributes); + rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.providers, rpcClient.codec, + rpcClient.compressor, rpcClient.cellBlockBuilder, rpcClient.metrics, + rpcClient.connectionAttributes); this.rpcClient = rpcClient; this.connectionHeaderPreamble = getConnectionHeaderPreamble(); ConnectionHeader header = getConnectionHeader(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java index b7856c089133..d65c19e34edd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java @@ -106,8 +106,9 @@ class NettyRpcConnection extends RpcConnection { NettyRpcConnection(NettyRpcClient rpcClient, ConnectionId remoteId) throws IOException { super(rpcClient.conf, AbstractRpcClient.WHEEL_TIMER, remoteId, rpcClient.clusterId, - rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.codec, rpcClient.compressor, - rpcClient.cellBlockBuilder, rpcClient.metrics, rpcClient.connectionAttributes); + rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.providers, rpcClient.codec, + rpcClient.compressor, rpcClient.cellBlockBuilder, rpcClient.metrics, + rpcClient.connectionAttributes); this.rpcClient = rpcClient; this.eventLoop = rpcClient.group.next(); byte[] connectionHeaderPreamble = getConnectionHeaderPreamble(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java index 8017e99ec4ff..2ec5a7ee1a39 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java @@ -119,9 +119,9 @@ abstract class RpcConnection { private String lastSucceededServerPrincipal; protected RpcConnection(Configuration conf, HashedWheelTimer timeoutTimer, ConnectionId remoteId, - String clusterId, boolean isSecurityEnabled, Codec codec, CompressionCodec compressor, - CellBlockBuilder cellBlockBuilder, MetricsConnection metrics, - Map connectionAttributes) throws IOException { + String clusterId, boolean isSecurityEnabled, SaslClientAuthenticationProviders providers, + Codec codec, CompressionCodec compressor, CellBlockBuilder cellBlockBuilder, + MetricsConnection metrics, Map connectionAttributes) throws IOException { this.timeoutTimer = timeoutTimer; this.codec = codec; this.compressor = compressor; @@ -134,8 +134,6 @@ protected RpcConnection(Configuration conf, HashedWheelTimer timeoutTimer, Conne this.useSasl = isSecurityEnabled; // Choose the correct Token and AuthenticationProvider for this client to use - SaslClientAuthenticationProviders providers = - SaslClientAuthenticationProviders.getInstance(conf); Pair> pair; if (useSasl && securityInfo != null) { pair = providers.selectProvider(clusterId, ticket); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index 1afb15c0ac61..4bdf5e5af049 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -206,37 +206,6 @@ private static Quotas getQuotas(final Connection connection, final byte[] rowKey return quotasFromData(result.getValue(QUOTA_FAMILY_INFO, qualifier)); } - public static Get makeGetForTableQuotas(final TableName table) { - Get get = new Get(getTableRowKey(table)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForNamespaceQuotas(final String namespace) { - Get get = new Get(getNamespaceRowKey(namespace)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForRegionServerQuotas(final String regionServer) { - Get get = new Get(getRegionServerRowKey(regionServer)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForUserQuotas(final String user, final Iterable tables, - final Iterable namespaces) { - Get get = new Get(getUserRowKey(user)); - get.addColumn(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); - for (final TableName table : tables) { - get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserTable(table)); - } - for (final String ns : namespaces) { - get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserNamespace(ns)); - } - return get; - } - public static Scan makeScan(final QuotaFilter filter) { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java index d4ab38f5bf73..b08179a27a58 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java @@ -205,4 +205,15 @@ protected static long timeFromString(String timeDiff) { } return -1; } + + /** + * There is little value in an RpcThrottlingException having a stack trace, since its cause is + * well understood without one. When a RegionServer is under heavy load and needs to serve many + * RpcThrottlingExceptions, skipping fillInStackTrace() will save CPU time and allocations, both + * here and later when the exception must be serialized over the wire. + */ + @Override + public synchronized Throwable fillInStackTrace() { + return this; + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/AuthenticationProviderSelector.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/AuthenticationProviderSelector.java index cdd1fdb381f6..4653217b90d2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/AuthenticationProviderSelector.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/AuthenticationProviderSelector.java @@ -32,9 +32,10 @@ public interface AuthenticationProviderSelector { /** - * Initializes the implementation with configuration and a set of providers available. This method - * should be called exactly once per implementation prior to calling - * {@link #selectProvider(String, User)}. + * Initializes the implementation with configuration and a set of providers available. + *

+ * This method is called once upon construction, before {@link #selectProvider(String, User)} is + * invoked. */ void configure(Configuration conf, Collection availableProviders); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/SaslClientAuthenticationProviders.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/SaslClientAuthenticationProviders.java index a78ff3386a46..a0ad1b405d0b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/SaslClientAuthenticationProviders.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/SaslClientAuthenticationProviders.java @@ -48,16 +48,36 @@ public final class SaslClientAuthenticationProviders { public static final String SELECTOR_KEY = "hbase.client.sasl.provider.class"; public static final String EXTRA_PROVIDERS_KEY = "hbase.client.sasl.provider.extras"; - private static final AtomicReference providersRef = + private static final AtomicReference PROVIDER_REF = new AtomicReference<>(); private final Collection providers; private final AuthenticationProviderSelector selector; - private SaslClientAuthenticationProviders(Collection providers, - AuthenticationProviderSelector selector) { - this.providers = providers; - this.selector = selector; + /** + * Creates a new instance of SaslClientAuthenticationProviders. + * @param conf the configuration to use for loading providers and selector + */ + public SaslClientAuthenticationProviders(Configuration conf) { + ServiceLoader loader = + ServiceLoader.load(SaslClientAuthenticationProvider.class, + SaslClientAuthenticationProviders.class.getClassLoader()); + HashMap providerMap = new HashMap<>(); + for (SaslClientAuthenticationProvider provider : loader) { + addProviderIfNotExists(provider, providerMap); + } + + addExplicitProviders(conf, providerMap); + + providers = Collections.unmodifiableCollection(providerMap.values()); + + if (LOG.isTraceEnabled()) { + String loadedProviders = providers.stream().map((provider) -> provider.getClass().getName()) + .collect(Collectors.joining(", ")); + LOG.trace("Found SaslClientAuthenticationProviders {}", loadedProviders); + } + + selector = instantiateSelector(conf, providers); } /** @@ -69,12 +89,16 @@ public int getNumRegisteredProviders() { /** * Returns a singleton instance of {@link SaslClientAuthenticationProviders}. + * @deprecated Since 2.5.14 and 2.6.4, will be removed in newer minor release lines. This class + * should not be singleton, please do not use it any more. see HBASE-29144 for more + * details. */ + @Deprecated public static synchronized SaslClientAuthenticationProviders getInstance(Configuration conf) { - SaslClientAuthenticationProviders providers = providersRef.get(); + SaslClientAuthenticationProviders providers = PROVIDER_REF.get(); if (providers == null) { - providers = instantiate(conf); - providersRef.set(providers); + providers = new SaslClientAuthenticationProviders(conf); + PROVIDER_REF.set(providers); } return providers; @@ -82,9 +106,13 @@ public static synchronized SaslClientAuthenticationProviders getInstance(Configu /** * Removes the cached singleton instance of {@link SaslClientAuthenticationProviders}. + * @deprecated Since 2.5.14 and 2.6.4, will be removed in newer minor release lines. This class + * should not be singleton, please do not use it any more. see HBASE-29144 for more + * details. */ + @Deprecated public static synchronized void reset() { - providersRef.set(null); + PROVIDER_REF.set(null); } /** @@ -93,7 +121,7 @@ public static synchronized void reset() { */ static void addProviderIfNotExists(SaslClientAuthenticationProvider provider, HashMap providers) { - Byte code = provider.getSaslAuthMethod().getCode(); + byte code = provider.getSaslAuthMethod().getCode(); SaslClientAuthenticationProvider existingProvider = providers.get(code); if (existingProvider != null) { throw new RuntimeException("Already registered authentication provider with " + code + " " @@ -105,7 +133,7 @@ static void addProviderIfNotExists(SaslClientAuthenticationProvider provider, /** * Instantiates the ProviderSelector implementation from the provided configuration. */ - static AuthenticationProviderSelector instantiateSelector(Configuration conf, + private static AuthenticationProviderSelector instantiateSelector(Configuration conf, Collection providers) { Class clz = conf.getClass(SELECTOR_KEY, BuiltInProviderSelector.class, AuthenticationProviderSelector.class); @@ -161,34 +189,6 @@ static void addExplicitProviders(Configuration conf, } } - /** - * Instantiates all client authentication providers and returns an instance of - * {@link SaslClientAuthenticationProviders}. - */ - static SaslClientAuthenticationProviders instantiate(Configuration conf) { - ServiceLoader loader = - ServiceLoader.load(SaslClientAuthenticationProvider.class, - SaslClientAuthenticationProviders.class.getClassLoader()); - HashMap providerMap = new HashMap<>(); - for (SaslClientAuthenticationProvider provider : loader) { - addProviderIfNotExists(provider, providerMap); - } - - addExplicitProviders(conf, providerMap); - - Collection providers = - Collections.unmodifiableCollection(providerMap.values()); - - if (LOG.isTraceEnabled()) { - String loadedProviders = providers.stream().map((provider) -> provider.getClass().getName()) - .collect(Collectors.joining(", ")); - LOG.trace("Found SaslClientAuthenticationProviders {}", loadedProviders); - } - - AuthenticationProviderSelector selector = instantiateSelector(conf, providers); - return new SaslClientAuthenticationProviders(providers, selector); - } - /** * Returns the provider and token pair for SIMPLE authentication. This method is a "hack" while * SIMPLE authentication for HBase does not flow through the SASL codepath. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 63bb5ba9f389..02c31d19a637 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -141,6 +141,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RegionSpecifierAndState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RunCatalogScanRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RunCleanerChoreRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetBalancerRunningRequest; @@ -1318,6 +1319,31 @@ public static ModifyTableRequest buildModifyTableRequest(final TableName tableNa return builder.build(); } + /** + * Creates a protocol buffer ReopenTableRegionsRequest + * @param tableName table whose regions to reopen + * @param regionNames specific regions to reopen (empty = all regions) + * @param nonceGroup nonce group + * @param nonce nonce + * @return a ReopenTableRegionsRequest + */ + public static ReopenTableRegionsRequest buildReopenTableRegionsRequest(final TableName tableName, + final List regionNames, final long nonceGroup, final long nonce) { + ReopenTableRegionsRequest.Builder builder = ReopenTableRegionsRequest.newBuilder(); + builder.setTableName(ProtobufUtil.toProtoTableName(tableName)); + + if (regionNames != null && !regionNames.isEmpty()) { + for (byte[] regionName : regionNames) { + builder.addRegionNames(UnsafeByteOperations.unsafeWrap(regionName)); + } + } + + builder.setNonceGroup(nonceGroup); + builder.setNonce(nonce); + + return builder.build(); + } + public static ModifyTableStoreFileTrackerRequest buildModifyTableStoreFileTrackerRequest( final TableName tableName, final String dstSFT, final long nonceGroup, final long nonce) { ModifyTableStoreFileTrackerRequest.Builder builder = diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java index bd1f4be6f0e9..040317c238e0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java @@ -166,8 +166,12 @@ public static org.apache.hadoop.hbase.client.MultiResponse getResults(final Mult if (!r.isEmpty()) { result = r; } - results.add(regionName, roe.getIndex(), - new CheckAndMutateResult(actionResult.getProcessed(), result)); + CheckAndMutateResult camResult = + new CheckAndMutateResult(actionResult.getProcessed(), result); + if (roe.hasMetrics()) { + camResult.setMetrics(ProtobufUtil.toQueryMetrics(roe.getMetrics())); + } + results.add(regionName, roe.getIndex(), camResult); } } else { if (actionResult.hasProcessed()) { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java index fc05d9eec523..3687f20e8265 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.exceptions.HBaseException; @@ -31,29 +32,19 @@ import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.PrettyPrinter; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Tests the HColumnDescriptor with appropriate arguments. * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0 together with * {@link HColumnDescriptor}. */ -@Category({ MiscTests.class, SmallTests.class }) @Deprecated +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestHColumnDescriptor { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHColumnDescriptor.class); - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - @Test public void testPb() throws DeserializationException { HColumnDescriptor hcd = new HColumnDescriptor(new HColumnDescriptor(HConstants.CATALOG_FAMILY) @@ -104,9 +95,9 @@ public void testPb() throws DeserializationException { */ @Test public void testHColumnDescriptorShouldThrowIAEWhenFamilyNameEmpty() throws Exception { - expectedEx.expect(IllegalArgumentException.class); - expectedEx.expectMessage("Column Family name can not be empty"); - new HColumnDescriptor(Bytes.toBytes("")); + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> new HColumnDescriptor("")); + assertEquals("Column Family name can not be empty", exception.getMessage()); } /** @@ -160,30 +151,30 @@ public void testSetTimeToLive() throws HBaseException { ttl = "50000"; desc.setTimeToLive(ttl); - Assert.assertEquals(50000, desc.getTimeToLive()); + assertEquals(50000, desc.getTimeToLive()); ttl = "50000 seconds"; desc.setTimeToLive(ttl); - Assert.assertEquals(50000, desc.getTimeToLive()); + assertEquals(50000, desc.getTimeToLive()); ttl = ""; desc.setTimeToLive(ttl); - Assert.assertEquals(0, desc.getTimeToLive()); + assertEquals(0, desc.getTimeToLive()); ttl = "FOREVER"; desc.setTimeToLive(ttl); - Assert.assertEquals(HConstants.FOREVER, desc.getTimeToLive()); + assertEquals(HConstants.FOREVER, desc.getTimeToLive()); ttl = "1 HOUR 10 minutes 1 second"; desc.setTimeToLive(ttl); - Assert.assertEquals(4201, desc.getTimeToLive()); + assertEquals(4201, desc.getTimeToLive()); ttl = "500 Days 23 HOURS"; desc.setTimeToLive(ttl); - Assert.assertEquals(43282800, desc.getTimeToLive()); + assertEquals(43282800, desc.getTimeToLive()); ttl = "43282800 SECONDS (500 Days 23 hours)"; desc.setTimeToLive(ttl); - Assert.assertEquals(43282800, desc.getTimeToLive()); + assertEquals(43282800, desc.getTimeToLive()); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java index cdf043dad766..27db8d0144c4 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java @@ -17,10 +17,11 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.Arrays; @@ -31,11 +32,10 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,25 +44,26 @@ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0 together with * {@link HTableDescriptor}. */ -@Category({ MiscTests.class, SmallTests.class }) @Deprecated +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestHTableDescriptor { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHTableDescriptor.class); - private static final Logger LOG = LoggerFactory.getLogger(TestHTableDescriptor.class); - @Rule - public TestName name = new TestName(); + private String methodName; + + @BeforeEach + public void setUpBeforeEach(TestInfo testInfo) { + methodName = testInfo.getTestMethod().get().getName(); + } - @Test(expected = IOException.class) + @Test public void testAddCoprocessorTwice() throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); String cpName = "a.b.c.d"; htd.addCoprocessor(cpName); - htd.addCoprocessor(cpName); + assertThrows(IOException.class, () -> htd.addCoprocessor(cpName)); } @Test @@ -120,7 +121,7 @@ public void testPb() throws DeserializationException, IOException { */ @Test public void testGetSetRemoveCP() throws Exception { - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(methodName)); // simple CP String className = "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver"; // add and check that it is present @@ -137,7 +138,7 @@ public void testGetSetRemoveCP() throws Exception { */ @Test public void testSetListRemoveCP() throws Exception { - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(methodName)); // simple CP String className1 = "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver"; String className2 = "org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver"; @@ -173,7 +174,7 @@ public void testSetListRemoveCP() throws Exception { */ @Test public void testAddGetRemoveString() { - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(methodName)); String key = "Some"; String value = "value"; desc.setValue(key, value); @@ -234,8 +235,8 @@ public void testIllegalZooKeeperName() { public void testLegalHTableNamesRegex() { for (String tn : legalTableNames) { TableName tName = TableName.valueOf(tn); - assertTrue("Testing: '" + tn + "'", - Pattern.matches(TableName.VALID_USER_TABLE_REGEX, tName.getNameAsString())); + assertTrue(Pattern.matches(TableName.VALID_USER_TABLE_REGEX, tName.getNameAsString()), + "Testing: '" + tn + "'"); } } @@ -252,7 +253,7 @@ public void testIllegalHTableNamesRegex() { */ @Test public void testGetMaxFileSize() { - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(methodName)); assertEquals(-1, desc.getMaxFileSize()); desc.setMaxFileSize(1111L); assertEquals(1111L, desc.getMaxFileSize()); @@ -263,7 +264,7 @@ public void testGetMaxFileSize() { */ @Test public void testGetMemStoreFlushSize() { - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(methodName)); assertEquals(-1, desc.getMemStoreFlushSize()); desc.setMemStoreFlushSize(1111L); assertEquals(1111L, desc.getMemStoreFlushSize()); @@ -274,7 +275,7 @@ public void testGetMemStoreFlushSize() { */ @Test public void testAddGetRemoveConfiguration() throws Exception { - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(methodName)); String key = "Some"; String value = "value"; desc.setConfiguration(key, value); @@ -297,7 +298,7 @@ public void testClassMethodsAreBuilderStyle() { @Test public void testModifyFamily() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(methodName)); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); @@ -313,17 +314,17 @@ public void testModifyFamily() { assertEquals(1, htd.getFamily(familyName).getDFSReplication()); } - @Test(expected = IllegalArgumentException.class) + @Test public void testModifyInexistentFamily() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(methodName)); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); - htd.modifyFamily(hcd); + assertThrows(IllegalArgumentException.class, () -> htd.modifyFamily(hcd)); } - @Test(expected = IllegalArgumentException.class) + @Test public void testAddDuplicateFamilies() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(methodName)); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); @@ -331,12 +332,13 @@ public void testAddDuplicateFamilies() { assertEquals(1000, htd.getFamily(familyName).getBlocksize()); hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(2000); - htd.addFamily(hcd); + HColumnDescriptor finalHcd = hcd; + assertThrows(IllegalArgumentException.class, () -> htd.addFamily(finalHcd)); } @Test public void testPriority() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(methodName)); htd.setPriority(42); assertEquals(42, htd.getPriority()); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java index d87f3b3cb042..a53673f4ba55 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java @@ -17,25 +17,21 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestRegionLocations { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionLocations.class); - ServerName sn0 = ServerName.valueOf("host0", 10, 10); ServerName sn1 = ServerName.valueOf("host1", 10, 10); ServerName sn2 = ServerName.valueOf("host2", 10, 10); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImplTest.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImplTest.java index 965259983e69..4f4d3f4ffa43 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImplTest.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImplTest.java @@ -17,26 +17,22 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException; -import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(SmallTests.TAG) +@Tag(SmallTests.TAG) public class RpcRetryingCallerImplTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(RpcRetryingCallerImplTest.class); - @Test public void itTranslatesRemoteExceptionFromServiceException() throws DoNotRetryIOException { String message = "CDE for test"; @@ -44,7 +40,7 @@ public void itTranslatesRemoteExceptionFromServiceException() throws DoNotRetryI new RemoteWithExtrasException(CallDroppedException.class.getName(), message, false)); Throwable result = RpcRetryingCallerImpl.translateException(exception); - Assert.assertTrue("Expect unwrap CallDroppedException", result instanceof CallDroppedException); - Assert.assertEquals(message, result.getMessage()); + assertTrue(result instanceof CallDroppedException, "Expect unwrap CallDroppedException"); + assertEquals(message, result.getMessage()); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminRpcPriority.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminRpcPriority.java index 4475a29123a8..0002ce8ed4f8 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminRpcPriority.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminRpcPriority.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -40,12 +39,10 @@ import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -69,13 +66,10 @@ /** * Confirm that we will set the priority in {@link HBaseRpcController} for several admin operations. */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestAsyncAdminRpcPriority { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncAdminRpcPriority.class); - private static Configuration CONF = HBaseConfiguration.create(); private MasterService.Interface masterStub; @@ -84,11 +78,11 @@ public class TestAsyncAdminRpcPriority { private AsyncConnection conn; - @Rule - public TestName name = new TestName(); + public String name; - @Before - public void setUp() throws IOException { + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException { + name = testInfo.getTestMethod().get().getName(); masterStub = mock(MasterService.Interface.class); adminStub = mock(AdminService.Interface.class); doAnswer(new Answer() { @@ -171,10 +165,8 @@ public boolean matches(HBaseRpcController controller) { @Test public void testCreateNormalTable() { - conn.getAdmin() - .createTable(TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build()) - .join(); + conn.getAdmin().createTable(TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build()).join(); verify(masterStub, times(1)).createTable(assertPriority(NORMAL_QOS), any(CreateTableRequest.class), any()); } @@ -184,9 +176,9 @@ public void testCreateNormalTable() { @Test public void testCreateSystemTable() { conn.getAdmin() - .createTable(TableDescriptorBuilder - .newBuilder(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build()) + .createTable( + TableDescriptorBuilder.newBuilder(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build()) .join(); verify(masterStub, times(1)).createTable(assertPriority(SYSTEMTABLE_QOS), any(CreateTableRequest.class), any()); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncBatchRpcRetryingCaller.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncBatchRpcRetryingCaller.java index 0d37ee454b12..ab4c1fe3d587 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncBatchRpcRetryingCaller.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncBatchRpcRetryingCaller.java @@ -20,7 +20,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.startsWith; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -32,7 +32,6 @@ import java.io.IOException; import java.util.IdentityHashMap; import java.util.concurrent.atomic.AtomicReference; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -41,24 +40,20 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestAsyncBatchRpcRetryingCaller { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncBatchRpcRetryingCaller.class); - private org.apache.logging.log4j.core.Appender mockAppender; - @Before + @BeforeEach public void setUp() { mockAppender = mock(org.apache.logging.log4j.core.Appender.class); when(mockAppender.getName()).thenReturn("mockAppender"); @@ -68,7 +63,7 @@ public void setUp() { } - @After + @AfterEach public void tearDown() { ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager .getLogger(AsyncBatchRpcRetryingCaller.class)).removeAppender(mockAppender); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionConfiguration.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionConfiguration.java index f4c9fc059bd2..5fb354a850cf 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionConfiguration.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionConfiguration.java @@ -17,30 +17,25 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * See HBASE-24513. */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestAsyncConnectionConfiguration { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncConnectionConfiguration.class); - @Test public void itHandlesDeprecatedPauseForCQTBE() { Configuration conf = new Configuration(); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionTracing.java index 2292c0d09e5b..a73c349fd1b8 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionTracing.java @@ -17,16 +17,15 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; import io.opentelemetry.sdk.trace.data.SpanData; import java.io.IOException; import java.util.concurrent.CompletableFuture; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Waiter; @@ -35,22 +34,18 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.trace.HBaseSemanticAttributes; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestAsyncConnectionTracing { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncConnectionTracing.class); - private static Configuration CONF = HBaseConfiguration.create(); private ServerName masterServer = @@ -58,10 +53,10 @@ public class TestAsyncConnectionTracing { private AsyncConnection conn; - @Rule - public OpenTelemetryRule traceRule = OpenTelemetryRule.create(); + @RegisterExtension + public static final OpenTelemetryExtension traceRule = OpenTelemetryExtension.create(); - @Before + @BeforeEach public void setUp() throws IOException { User user = UserProvider.instantiate(CONF).getCurrent(); ConnectionRegistry registry = new DoNothingConnectionRegistry(CONF, user) { @@ -74,7 +69,7 @@ public CompletableFuture getActiveMaster() { conn = new AsyncConnectionImpl(CONF, registry, "test", user); } - @After + @AfterEach public void tearDown() throws IOException { Closeables.close(conn, true); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocatorFailFast.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocatorFailFast.java index 6380f1f6fb0f..fb7188ab3fa9 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocatorFailFast.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocatorFailFast.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.io.IOException; import java.util.concurrent.CompletableFuture; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.security.User; @@ -29,18 +30,14 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FutureUtils; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestAsyncMetaRegionLocatorFailFast { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncMetaRegionLocatorFailFast.class); - private static Configuration CONF = HBaseConfiguration.create(); private static AsyncMetaRegionLocator LOCATOR; @@ -57,14 +54,15 @@ public CompletableFuture getMetaRegionLocations() { } } - @BeforeClass + @BeforeAll public static void setUp() throws IOException { LOCATOR = new AsyncMetaRegionLocator( new FaultyConnectionRegistry(CONF, UserProvider.instantiate(CONF).getCurrent())); } - @Test(expected = DoNotRetryIOException.class) + @Test public void test() throws IOException { - FutureUtils.get(LOCATOR.getRegionLocations(RegionInfo.DEFAULT_REPLICA_ID, false)); + assertThrows(DoNotRetryIOException.class, + () -> FutureUtils.get(LOCATOR.getRegionLocations(RegionInfo.DEFAULT_REPLICA_ID, false))); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java index 245c802845a4..3b013f6bbc65 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.io.InterruptedIOException; @@ -51,7 +54,6 @@ import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.CallQueueTooBigException; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseServerException; import org.apache.hadoop.hbase.HConstants; @@ -74,24 +76,19 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Threads; -import org.junit.Assert; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; -@Category({ ClientTests.class, LargeTests.class }) +@Tag(ClientTests.TAG) +@Tag(LargeTests.TAG) public class TestAsyncProcess { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncProcess.class); - private static final Logger LOG = LoggerFactory.getLogger(TestAsyncProcess.class); private static final TableName DUMMY_TABLE = TableName.valueOf("DUMMY_TABLE"); private static final byte[] DUMMY_BYTES_1 = Bytes.toBytes("DUMMY_BYTES_1"); @@ -132,7 +129,7 @@ public class TestAsyncProcess { private int RPC_TIMEOUT; private int OPERATION_TIMEOUT; - @Before + @BeforeEach public void beforeEach() { this.CONF = new Configuration(); CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, NB_RETRIES); @@ -771,7 +768,7 @@ public void testSubmit() throws Exception { puts.add(createPut(1, true)); ap.submit(null, DUMMY_TABLE, puts, false, null, false); - Assert.assertTrue(puts.isEmpty()); + assertTrue(puts.isEmpty()); } @Test @@ -790,9 +787,9 @@ public void update(byte[] region, byte[] row, Object result) { puts.add(createPut(1, true)); final AsyncRequestFuture ars = ap.submit(null, DUMMY_TABLE, puts, false, cb, false); - Assert.assertTrue(puts.isEmpty()); + assertTrue(puts.isEmpty()); ars.waitUntilDone(); - Assert.assertEquals(1, updateCalled.get()); + assertEquals(1, updateCalled.get()); } @Test @@ -811,11 +808,11 @@ public void testSubmitBusyRegion() throws Exception { ap.incTaskCounters(Collections.singleton(hri1.getRegionName()), sn); } ap.submit(null, DUMMY_TABLE, puts, false, null, false); - Assert.assertEquals(puts.size(), 1); + assertEquals(puts.size(), 1); ap.decTaskCounters(Collections.singleton(hri1.getRegionName()), sn); ap.submit(null, DUMMY_TABLE, puts, false, null, false); - Assert.assertEquals(0, puts.size()); + assertEquals(0, puts.size()); if (defaultClazz != null) { conn.getConfiguration().set(RequestControllerFactory.REQUEST_CONTROLLER_IMPL_CONF_KEY, defaultClazz); @@ -841,12 +838,12 @@ public void testSubmitBusyRegionServer() throws Exception { puts.add(createPut(2, true)); // <== new region, but the rs is ok ap.submit(null, DUMMY_TABLE, puts, false, null, false); - Assert.assertEquals(" puts=" + puts, 1, puts.size()); + assertEquals(1, puts.size(), " puts=" + puts); controller.taskCounterPerServer.put(sn2, new AtomicInteger(controller.maxConcurrentTasksPerServer - 1)); ap.submit(null, DUMMY_TABLE, puts, false, null, false); - Assert.assertTrue(puts.isEmpty()); + assertTrue(puts.isEmpty()); if (defaultClazz != null) { conn.getConfiguration().set(RequestControllerFactory.REQUEST_CONTROLLER_IMPL_CONF_KEY, defaultClazz); @@ -862,20 +859,20 @@ public void testFail() throws Exception { puts.add(p); AsyncRequestFuture ars = ap.submit(null, DUMMY_TABLE, puts, false, null, true); - Assert.assertEquals(0, puts.size()); + assertEquals(0, puts.size()); ars.waitUntilDone(); verifyResult(ars, false); - Assert.assertEquals(NB_RETRIES + 1, ap.callsCt.get()); + assertEquals(NB_RETRIES + 1, ap.callsCt.get()); - Assert.assertEquals(1, ars.getErrors().exceptions.size()); - Assert.assertTrue("was: " + ars.getErrors().exceptions.get(0), - failure.equals(ars.getErrors().exceptions.get(0))); - Assert.assertTrue("was: " + ars.getErrors().exceptions.get(0), - failure.equals(ars.getErrors().exceptions.get(0))); + assertEquals(1, ars.getErrors().exceptions.size()); + assertTrue(failure.equals(ars.getErrors().exceptions.get(0)), + "was: " + ars.getErrors().exceptions.get(0)); + assertTrue(failure.equals(ars.getErrors().exceptions.get(0)), + "was: " + ars.getErrors().exceptions.get(0)); - Assert.assertEquals(1, ars.getFailedOperations().size()); - Assert.assertTrue("was: " + ars.getFailedOperations().get(0), - p.equals(ars.getFailedOperations().get(0))); + assertEquals(1, ars.getFailedOperations().size()); + assertTrue(p.equals(ars.getFailedOperations().get(0)), + "was: " + ars.getFailedOperations().get(0)); } @Test @@ -898,7 +895,7 @@ public void testSubmitTrue() throws IOException { @Override public void run() { Threads.sleep(1000); - Assert.assertFalse(checkPoint.get()); // TODO: this is timing-dependent + assertFalse(checkPoint.get()); // TODO: this is timing-dependent ai.decrementAndGet(); controller.tasksInProgress.decrementAndGet(); checkPoint2.set(true); @@ -910,12 +907,12 @@ public void run() { puts.add(p); ap.submit(null, DUMMY_TABLE, puts, false, null, false); - Assert.assertFalse(puts.isEmpty()); + assertFalse(puts.isEmpty()); t.start(); ap.submit(null, DUMMY_TABLE, puts, true, null, false); - Assert.assertTrue(puts.isEmpty()); + assertTrue(puts.isEmpty()); checkPoint.set(true); while (!checkPoint2.get()) { @@ -937,20 +934,20 @@ public void testFailAndSuccess() throws Exception { puts.add(createPut(1, true)); AsyncRequestFuture ars = ap.submit(null, DUMMY_TABLE, puts, false, null, true); - Assert.assertTrue(puts.isEmpty()); + assertTrue(puts.isEmpty()); ars.waitUntilDone(); verifyResult(ars, false, true, true); - Assert.assertEquals(NB_RETRIES + 1, ap.callsCt.get()); + assertEquals(NB_RETRIES + 1, ap.callsCt.get()); ap.callsCt.set(0); - Assert.assertEquals(1, ars.getErrors().actions.size()); + assertEquals(1, ars.getErrors().actions.size()); puts.add(createPut(1, true)); // Wait for AP to be free. While ars might have the result, ap counters are decreased later. ap.waitForMaximumCurrentTasks(0, null); ars = ap.submit(null, DUMMY_TABLE, puts, false, null, true); - Assert.assertEquals(0, puts.size()); + assertEquals(0, puts.size()); ars.waitUntilDone(); - Assert.assertEquals(1, ap.callsCt.get()); + assertEquals(1, ap.callsCt.get()); verifyResult(ars, true); } @@ -966,9 +963,9 @@ public void testFlush() throws Exception { AsyncRequestFuture ars = ap.submit(null, DUMMY_TABLE, puts, false, null, true); ars.waitUntilDone(); verifyResult(ars, false, true, true); - Assert.assertEquals(NB_RETRIES + 1, ap.callsCt.get()); + assertEquals(NB_RETRIES + 1, ap.callsCt.get()); - Assert.assertEquals(1, ars.getFailedOperations().size()); + assertEquals(1, ars.getFailedOperations().size()); } @Test @@ -1052,7 +1049,7 @@ public void run() { try { ap.submit(null, DUMMY_TABLE, puts, false, null, false); - Assert.fail("We should have been interrupted."); + fail("We should have been interrupted."); } catch (InterruptedIOException expected) { } @@ -1074,7 +1071,7 @@ public void run() { long end = EnvironmentEdgeManager.currentTime(); // Adds 100 to secure us against approximate timing. - Assert.assertTrue(start + 100L + sleepTime > end); + assertTrue(start + 100L + sleepTime > end); if (defaultClazz != null) { conn.getConfiguration().set(RequestControllerFactory.REQUEST_CONTROLLER_IMPL_CONF_KEY, defaultClazz); @@ -1148,12 +1145,11 @@ public void testHTablePutSuccess() throws Exception { Put put = createPut(1, true); - Assert.assertEquals(conn.getConnectionConfiguration().getWriteBufferSize(), - ht.getWriteBufferSize()); - Assert.assertEquals(0, ht.getCurrentWriteBufferSize()); + assertEquals(conn.getConnectionConfiguration().getWriteBufferSize(), ht.getWriteBufferSize()); + assertEquals(0, ht.getCurrentWriteBufferSize()); ht.mutate(put); ht.flush(); - Assert.assertEquals(0, ht.getCurrentWriteBufferSize()); + assertEquals(0, ht.getCurrentWriteBufferSize()); } @Test @@ -1177,20 +1173,20 @@ private void checkPeriodicFlushParameters(ClusterConnection conn, MyAsyncProcess // The BufferedMutatorParams does nothing with the value bufferParam.setWriteBufferPeriodicFlushTimeoutMs(setTO); bufferParam.setWriteBufferPeriodicFlushTimerTickMs(setTT); - Assert.assertEquals(setTO, bufferParam.getWriteBufferPeriodicFlushTimeoutMs()); - Assert.assertEquals(setTT, bufferParam.getWriteBufferPeriodicFlushTimerTickMs()); + assertEquals(setTO, bufferParam.getWriteBufferPeriodicFlushTimeoutMs()); + assertEquals(setTT, bufferParam.getWriteBufferPeriodicFlushTimerTickMs()); // The BufferedMutatorImpl corrects illegal values (indirect via BufferedMutatorParams) BufferedMutatorImpl ht1 = new BufferedMutatorImpl(conn, bufferParam, ap); - Assert.assertEquals(expectTO, ht1.getWriteBufferPeriodicFlushTimeoutMs()); - Assert.assertEquals(expectTT, ht1.getWriteBufferPeriodicFlushTimerTickMs()); + assertEquals(expectTO, ht1.getWriteBufferPeriodicFlushTimeoutMs()); + assertEquals(expectTT, ht1.getWriteBufferPeriodicFlushTimerTickMs()); // The BufferedMutatorImpl corrects illegal values (direct via setter) BufferedMutatorImpl ht2 = new BufferedMutatorImpl(conn, createBufferedMutatorParams(ap, DUMMY_TABLE), ap); ht2.setWriteBufferPeriodicFlush(setTO, setTT); - Assert.assertEquals(expectTO, ht2.getWriteBufferPeriodicFlushTimeoutMs()); - Assert.assertEquals(expectTT, ht2.getWriteBufferPeriodicFlushTimerTickMs()); + assertEquals(expectTO, ht2.getWriteBufferPeriodicFlushTimeoutMs()); + assertEquals(expectTT, ht2.getWriteBufferPeriodicFlushTimerTickMs()); } @@ -1207,57 +1203,57 @@ public void testWriteBufferPeriodicFlushTimeoutMs() throws Exception { BufferedMutatorImpl ht = new BufferedMutatorImpl(conn, bufferParam, ap); // Verify if BufferedMutator has the right settings. - Assert.assertEquals(10000, ht.getWriteBufferSize()); - Assert.assertEquals(1, ht.getWriteBufferPeriodicFlushTimeoutMs()); - Assert.assertEquals(BufferedMutator.MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, + assertEquals(10000, ht.getWriteBufferSize()); + assertEquals(1, ht.getWriteBufferPeriodicFlushTimeoutMs()); + assertEquals(BufferedMutator.MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, ht.getWriteBufferPeriodicFlushTimerTickMs()); Put put = createPut(1, true); - Assert.assertEquals(0, ht.getExecutedWriteBufferPeriodicFlushes()); - Assert.assertEquals(0, ht.getCurrentWriteBufferSize()); + assertEquals(0, ht.getExecutedWriteBufferPeriodicFlushes()); + assertEquals(0, ht.getCurrentWriteBufferSize()); // ----- Insert, flush immediately, MUST NOT flush automatically ht.mutate(put); ht.flush(); Thread.sleep(1000); - Assert.assertEquals(0, ht.getExecutedWriteBufferPeriodicFlushes()); - Assert.assertEquals(0, ht.getCurrentWriteBufferSize()); + assertEquals(0, ht.getExecutedWriteBufferPeriodicFlushes()); + assertEquals(0, ht.getCurrentWriteBufferSize()); // ----- Insert, NO flush, MUST flush automatically ht.mutate(put); - Assert.assertEquals(0, ht.getExecutedWriteBufferPeriodicFlushes()); - Assert.assertTrue(ht.getCurrentWriteBufferSize() > 0); + assertEquals(0, ht.getExecutedWriteBufferPeriodicFlushes()); + assertTrue(ht.getCurrentWriteBufferSize() > 0); // The timerTick should fire every 100ms, so after twice that we must have // seen at least 1 tick and we should see an automatic flush Thread.sleep(200); - Assert.assertEquals(1, ht.getExecutedWriteBufferPeriodicFlushes()); - Assert.assertEquals(0, ht.getCurrentWriteBufferSize()); + assertEquals(1, ht.getExecutedWriteBufferPeriodicFlushes()); + assertEquals(0, ht.getCurrentWriteBufferSize()); // Ensure it does not flush twice Thread.sleep(200); - Assert.assertEquals(1, ht.getExecutedWriteBufferPeriodicFlushes()); - Assert.assertEquals(0, ht.getCurrentWriteBufferSize()); + assertEquals(1, ht.getExecutedWriteBufferPeriodicFlushes()); + assertEquals(0, ht.getCurrentWriteBufferSize()); // ----- DISABLE AUTO FLUSH, Insert, NO flush, MUST NOT flush automatically ht.disableWriteBufferPeriodicFlush(); ht.mutate(put); - Assert.assertEquals(1, ht.getExecutedWriteBufferPeriodicFlushes()); - Assert.assertTrue(ht.getCurrentWriteBufferSize() > 0); + assertEquals(1, ht.getExecutedWriteBufferPeriodicFlushes()); + assertTrue(ht.getCurrentWriteBufferSize() > 0); // Wait for at least 1 timerTick, we should see NO flushes. Thread.sleep(200); - Assert.assertEquals(1, ht.getExecutedWriteBufferPeriodicFlushes()); - Assert.assertTrue(ht.getCurrentWriteBufferSize() > 0); + assertEquals(1, ht.getExecutedWriteBufferPeriodicFlushes()); + assertTrue(ht.getCurrentWriteBufferSize() > 0); // Reenable periodic flushing, a flush seems to take about 1 second // so we wait for 2 seconds and it should have finished the flush. ht.setWriteBufferPeriodicFlush(1, 100); Thread.sleep(2000); - Assert.assertEquals(2, ht.getExecutedWriteBufferPeriodicFlushes()); - Assert.assertEquals(0, ht.getCurrentWriteBufferSize()); + assertEquals(2, ht.getExecutedWriteBufferPeriodicFlushes()); + assertEquals(0, ht.getCurrentWriteBufferSize()); } @Test @@ -1282,18 +1278,18 @@ public void testFailedPutAndNewPut() throws Exception { Put p = createPut(1, false); try { mutator.mutate(p); - Assert.fail(); + fail(); } catch (RetriesExhaustedWithDetailsException expected) { assertEquals(1, expected.getNumExceptions()); assertTrue(expected.getRow(0) == p); } // Let's do all the retries. ap.waitForMaximumCurrentTasks(0, null); - Assert.assertEquals(0, mutator.size()); + assertEquals(0, mutator.size()); // There is no global error so the new put should not fail mutator.mutate(createPut(1, true)); - Assert.assertEquals("the put should not been inserted.", 0, mutator.size()); + assertEquals(0, mutator.size(), "the put should not been inserted."); } @SuppressWarnings("SelfComparison") @@ -1334,17 +1330,17 @@ public void testBatch() throws IOException, InterruptedException { Object[] res = new Object[puts.size()]; try { ht.batch(puts, res); - Assert.fail(); + fail(); } catch (RetriesExhaustedException expected) { } - Assert.assertEquals(success, res[0]); - Assert.assertEquals(success, res[1]); - Assert.assertEquals(success, res[2]); - Assert.assertEquals(success, res[3]); - Assert.assertEquals(failure, res[4]); - Assert.assertEquals(success, res[5]); - Assert.assertEquals(failure, res[6]); + assertEquals(success, res[0]); + assertEquals(success, res[1]); + assertEquals(success, res[2]); + assertEquals(success, res[3]); + assertEquals(failure, res[4]); + assertEquals(success, res[5]); + assertEquals(failure, res[6]); } @Test @@ -1356,8 +1352,8 @@ public void testErrorsServers() throws IOException { BufferedMutatorImpl mutator = new BufferedMutatorImpl(conn, bufferParam, ap); configuration.setBoolean(ConnectionImplementation.RETRIES_BY_SERVER_KEY, true); - Assert.assertNotNull(ap.createServerErrorTracker()); - Assert.assertTrue(ap.serverTrackerTimeout > 200L); + assertNotNull(ap.createServerErrorTracker()); + assertTrue(ap.serverTrackerTimeout > 200L); ap.serverTrackerTimeout = 1L; Put p = createPut(1, false); @@ -1365,13 +1361,13 @@ public void testErrorsServers() throws IOException { try { mutator.flush(); - Assert.fail(); + fail(); } catch (RetriesExhaustedWithDetailsException expected) { assertEquals(1, expected.getNumExceptions()); assertTrue(expected.getRow(0) == p); } // Checking that the ErrorsServers came into play and didn't make us stop immediately - Assert.assertEquals(NB_RETRIES + 1, ap.callsCt.get()); + assertEquals(NB_RETRIES + 1, ap.callsCt.get()); } @Test @@ -1419,20 +1415,20 @@ public void testErrors() throws IOException { BufferedMutatorParams bufferParam = createBufferedMutatorParams(ap, DUMMY_TABLE); BufferedMutatorImpl mutator = new BufferedMutatorImpl(conn, bufferParam, ap); - Assert.assertNotNull(ap.createServerErrorTracker()); + assertNotNull(ap.createServerErrorTracker()); Put p = createPut(1, true); mutator.mutate(p); try { mutator.flush(); - Assert.fail(); + fail(); } catch (RetriesExhaustedWithDetailsException expected) { assertEquals(1, expected.getNumExceptions()); assertTrue(expected.getRow(0) == p); } // Checking that the ErrorsServers came into play and didn't make us stop immediately - Assert.assertEquals(NB_RETRIES + 1, ap.callsCt.get()); + assertEquals(NB_RETRIES + 1, ap.callsCt.get()); } @Test @@ -1442,19 +1438,19 @@ public void testCallQueueTooLarge() throws IOException { new AsyncProcessWithFailure(conn, CONF, new CallQueueTooBigException()); BufferedMutatorParams bufferParam = createBufferedMutatorParams(ap, DUMMY_TABLE); BufferedMutatorImpl mutator = new BufferedMutatorImpl(conn, bufferParam, ap); - Assert.assertNotNull(ap.createServerErrorTracker()); + assertNotNull(ap.createServerErrorTracker()); Put p = createPut(1, true); mutator.mutate(p); try { mutator.flush(); - Assert.fail(); + fail(); } catch (RetriesExhaustedWithDetailsException expected) { assertEquals(1, expected.getNumExceptions()); assertTrue(expected.getRow(0) == p); } // Checking that the ErrorsServers came into play and didn't make us stop immediately - Assert.assertEquals(NB_RETRIES + 1, ap.callsCt.get()); + assertEquals(NB_RETRIES + 1, ap.callsCt.get()); } /** @@ -1482,15 +1478,15 @@ public void testThreadCreation() throws Exception { ht.multiAp = ap; ht.batch(gets, null); - Assert.assertEquals(NB_REGS, ap.nbActions.get()); - Assert.assertEquals("1 multi response per server", 2, ap.nbMultiResponse.get()); - Assert.assertEquals("1 thread per server", 2, con.nbThreads.get()); + assertEquals(NB_REGS, ap.nbActions.get()); + assertEquals(2, ap.nbMultiResponse.get(), "1 multi response per server"); + assertEquals(2, con.nbThreads.get(), "1 thread per server"); int nbReg = 0; for (int i = 0; i < NB_REGS; i++) { if (con.usedRegions[i]) nbReg++; } - Assert.assertEquals("nbReg=" + nbReg, NB_REGS, nbReg); + assertEquals(NB_REGS, nbReg, "nbReg=" + nbReg); } @Test @@ -1504,7 +1500,7 @@ public void testReplicaReplicaSuccess() throws Exception { .setRowAccess(rows).setResults(new Object[3]).setSubmittedRows(SubmittedRows.ALL).build(); AsyncRequestFuture ars = ap.submit(task); verifyReplicaResult(ars, RR.TRUE, RR.TRUE, RR.FALSE); - Assert.assertEquals(2, ap.getReplicaCallCount()); + assertEquals(2, ap.getReplicaCallCount()); } @Test @@ -1517,7 +1513,7 @@ public void testReplicaPrimarySuccessWoReplicaCalls() throws Exception { .setRowAccess(rows).setResults(new Object[3]).setSubmittedRows(SubmittedRows.ALL).build(); AsyncRequestFuture ars = ap.submit(task); verifyReplicaResult(ars, RR.FALSE, RR.FALSE, RR.FALSE); - Assert.assertEquals(0, ap.getReplicaCallCount()); + assertEquals(0, ap.getReplicaCallCount()); } @Test @@ -1531,8 +1527,8 @@ public void testReplicaParallelCallsSucceed() throws Exception { AsyncRequestFuture ars = ap.submit(task); verifyReplicaResult(ars, RR.DONT_CARE, RR.DONT_CARE); long replicaCalls = ap.getReplicaCallCount(); - Assert.assertTrue(replicaCalls >= 0); - Assert.assertTrue(replicaCalls <= 2); + assertTrue(replicaCalls >= 0); + assertTrue(replicaCalls <= 2); } @Test @@ -1548,7 +1544,7 @@ public void testReplicaPartialReplicaCall() throws Exception { .setRowAccess(rows).setResults(new Object[2]).setSubmittedRows(SubmittedRows.ALL).build(); AsyncRequestFuture ars = ap.submit(task); verifyReplicaResult(ars, RR.FALSE, RR.TRUE); - Assert.assertEquals(1, ap.getReplicaCallCount()); + assertEquals(1, ap.getReplicaCallCount()); } @Test @@ -1564,7 +1560,7 @@ public void testReplicaMainFailsBeforeReplicaCalls() throws Exception { .setRowAccess(rows).setResults(new Object[2]).setSubmittedRows(SubmittedRows.ALL).build(); AsyncRequestFuture ars = ap.submit(task); verifyReplicaResult(ars, RR.FAILED, RR.FAILED); - Assert.assertEquals(0, ap.getReplicaCallCount()); + assertEquals(0, ap.getReplicaCallCount()); } @Test @@ -1579,7 +1575,7 @@ public void testReplicaReplicaSuccessWithParallelFailures() throws Exception { .setRowAccess(rows).setResults(new Object[2]).setSubmittedRows(SubmittedRows.ALL).build(); AsyncRequestFuture ars = ap.submit(task); verifyReplicaResult(ars, RR.TRUE, RR.TRUE); - Assert.assertEquals(2, ap.getReplicaCallCount()); + assertEquals(2, ap.getReplicaCallCount()); } @Test @@ -1595,9 +1591,9 @@ public void testReplicaAllCallsFailForOneRegion() throws Exception { AsyncRequestFuture ars = ap.submit(task); verifyReplicaResult(ars, RR.FAILED, RR.FALSE); // We should get 3 exceptions, for main + 2 replicas for DUMMY_BYTES_1 - Assert.assertEquals(3, ars.getErrors().getNumExceptions()); + assertEquals(3, ars.getErrors().getNumExceptions()); for (int i = 0; i < ars.getErrors().getNumExceptions(); ++i) { - Assert.assertArrayEquals(DUMMY_BYTES_1, ars.getErrors().getRow(i).getRow()); + assertArrayEquals(DUMMY_BYTES_1, ars.getErrors().getRow(i).getRow()); } } @@ -1638,9 +1634,9 @@ private static List makeTimelineGets(byte[]... rows) { private void verifyResult(AsyncRequestFuture ars, boolean... expected) throws Exception { Object[] actual = ars.getResults(); - Assert.assertEquals(expected.length, actual.length); + assertEquals(expected.length, actual.length); for (int i = 0; i < expected.length; ++i) { - Assert.assertEquals(expected[i], !(actual[i] instanceof Throwable)); + assertEquals(expected[i], !(actual[i] instanceof Throwable)); } } @@ -1654,13 +1650,13 @@ private enum RR { private void verifyReplicaResult(AsyncRequestFuture ars, RR... expecteds) throws Exception { Object[] actuals = ars.getResults(); - Assert.assertEquals(expecteds.length, actuals.length); + assertEquals(expecteds.length, actuals.length); for (int i = 0; i < expecteds.length; ++i) { Object actual = actuals[i]; RR expected = expecteds[i]; - Assert.assertEquals(actual.toString(), expected == RR.FAILED, actual instanceof Throwable); + assertEquals(expected == RR.FAILED, actual instanceof Throwable, actual.toString()); if (expected != RR.FAILED && expected != RR.DONT_CARE) { - Assert.assertEquals(expected == RR.TRUE, ((Result) actual).isStale()); + assertEquals(expected == RR.TRUE, ((Result) actual).isStale()); } } } @@ -1726,7 +1722,7 @@ public void testUncheckedException() throws Exception { .setOperationTimeout(OPERATION_TIMEOUT).setTableName(DUMMY_TABLE).setRowAccess(puts) .setSubmittedRows(SubmittedRows.NORMAL).build(); ap.submit(task); - Assert.assertTrue(puts.isEmpty()); + assertTrue(puts.isEmpty()); } /** @@ -1771,7 +1767,7 @@ private void testRetryPause(Configuration testConf, long expectedPause, BufferedMutatorParams bufferParam = createBufferedMutatorParams(ap, DUMMY_TABLE); BufferedMutatorImpl mutator = new BufferedMutatorImpl(conn, bufferParam, ap); - Assert.assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); + assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); Put p = createPut(1, true); mutator.mutate(p); @@ -1779,7 +1775,7 @@ private void testRetryPause(Configuration testConf, long expectedPause, long startTime = EnvironmentEdgeManager.currentTime(); try { mutator.flush(); - Assert.fail(); + fail(); } catch (RetriesExhaustedWithDetailsException expected) { assertEquals(1, expected.getNumExceptions()); assertTrue(expected.getRow(0) == p); @@ -1792,8 +1788,8 @@ private void testRetryPause(Configuration testConf, long expectedPause, actualSleep += (long) (expectedPause * 0.01f); } LOG.debug("Expected to sleep " + expectedSleep + "ms, actually slept " + actualSleep + "ms"); - Assert.assertTrue("Expected to sleep " + expectedSleep + " but actually " + actualSleep + "ms", - actualSleep >= expectedSleep); + assertTrue(actualSleep >= expectedSleep, + "Expected to sleep " + expectedSleep + " but actually " + actualSleep + "ms"); // check and confirm normal IOE will use the normal pause final long normalPause = @@ -1801,12 +1797,12 @@ private void testRetryPause(Configuration testConf, long expectedPause, ap = new AsyncProcessWithFailure(conn, testConf, new IOException()); bufferParam = createBufferedMutatorParams(ap, DUMMY_TABLE); mutator = new BufferedMutatorImpl(conn, bufferParam, ap); - Assert.assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); + assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); mutator.mutate(p); startTime = EnvironmentEdgeManager.currentTime(); try { mutator.flush(); - Assert.fail(); + fail(); } catch (RetriesExhaustedWithDetailsException expected) { assertEquals(1, expected.getNumExceptions()); assertTrue(expected.getRow(0) == p); @@ -1819,7 +1815,7 @@ private void testRetryPause(Configuration testConf, long expectedPause, // plus an additional pause to balance the program execution time expectedSleep += normalPause; LOG.debug("Expected to sleep " + expectedSleep + "ms, actually slept " + actualSleep + "ms"); - Assert.assertTrue("Slept for too long: " + actualSleep + "ms", actualSleep <= expectedSleep); + assertTrue(actualSleep <= expectedSleep, "Slept for too long: " + actualSleep + "ms"); } @Test @@ -1839,14 +1835,14 @@ public void testFastFailIfBackoffGreaterThanRemaining() throws IOException { createBufferedMutatorParams(ap, DUMMY_TABLE).operationTimeout(100); BufferedMutatorImpl mutator = new BufferedMutatorImpl(conn, bufferParam, ap); - Assert.assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); + assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); Put p = createPut(1, true); mutator.mutate(p); try { mutator.flush(); - Assert.fail(); + fail(); } catch (RetriesExhaustedWithDetailsException expected) { assertEquals(1, expected.getNumExceptions()); assertTrue(expected.getCause(0) instanceof OperationTimeoutExceededException); @@ -1886,9 +1882,9 @@ private void testRetryWithExceptionClearsMetaCache(boolean useRegionException) BufferedMutatorParams bufferParam = createBufferedMutatorParams(ap, DUMMY_TABLE); BufferedMutatorImpl mutator = new BufferedMutatorImpl(conn, bufferParam, ap); - Assert.assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); + assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); - Assert.assertEquals(conn.locateRegion(DUMMY_TABLE, DUMMY_BYTES_1, true, true).toString(), + assertEquals(conn.locateRegion(DUMMY_TABLE, DUMMY_BYTES_1, true, true).toString(), new RegionLocations(loc1).toString()); // simulate updateCachedLocations, by changing the loc for this row to loc3. only loc1 fails, @@ -1912,7 +1908,7 @@ private void testRetryWithExceptionClearsMetaCache(boolean useRegionException) mutator.flush(); // validate that we updated the location, as we expected - Assert.assertEquals(conn.locateRegion(DUMMY_TABLE, DUMMY_BYTES_1, true, true).toString(), + assertEquals(conn.locateRegion(DUMMY_TABLE, DUMMY_BYTES_1, true, true).toString(), new RegionLocations(loc3).toString()); // this is a given since the location updated, but validate that we called updateCachedLocations Mockito.verify(conn, Mockito.atLeastOnce()).updateCachedLocations(Mockito.eq(DUMMY_TABLE), diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcessWithRegionException.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcessWithRegionException.java index e63215c8ea54..f955eee4c3a3 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcessWithRegionException.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcessWithRegionException.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.io.InterruptedIOException; @@ -28,7 +28,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLocations; @@ -36,12 +35,11 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; /** @@ -58,13 +56,10 @@ * encounter this issue. We still backport the fix to branch-1.3 and branch-1.2 in case we ignore * some write paths. */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(LargeTests.TAG) public class TestAsyncProcessWithRegionException { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncProcessWithRegionException.class); - private static final Result EMPTY_RESULT = Result.create(null, true); private static final IOException IOE = new IOException("YOU CAN'T PASS"); private static final Configuration CONF = new Configuration(); @@ -84,7 +79,7 @@ public class TestAsyncProcessWithRegionException { private static final HRegionLocation REGION_LOCATION = new HRegionLocation(REGION_INFO, SERVER_NAME); - @BeforeClass + @BeforeAll public static void setUpBeforeClass() { // disable the retry CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java index a6fabe39affe..94d419cb6f61 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java @@ -33,14 +33,13 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; import io.opentelemetry.sdk.trace.data.SpanData; import java.io.IOException; import java.util.Arrays; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; @@ -54,35 +53,31 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.hamcrest.Matcher; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestAsyncRegionLocatorTracing { private static final Logger LOG = LoggerFactory.getLogger(TestAsyncRegionLocatorTracing.class); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncRegionLocatorTracing.class); - private static final Configuration CONF = HBaseConfiguration.create(); private AsyncConnectionImpl conn; private RegionLocations locs; - @Rule - public OpenTelemetryRule traceRule = OpenTelemetryRule.create(); + @RegisterExtension + public static final OpenTelemetryExtension traceRule = OpenTelemetryExtension.create(); - @Before + @BeforeEach public void setUp() throws IOException { RegionInfo metaRegionInfo = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).build(); locs = new RegionLocations( @@ -102,7 +97,7 @@ public CompletableFuture getMetaRegionLocations() { }, "test", user); } - @After + @AfterEach public void tearDown() throws IOException { Closeables.close(conn, true); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRpcPriority.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRpcPriority.java index 2bcf9c3ac124..2ae79ac429a4 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRpcPriority.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRpcPriority.java @@ -21,10 +21,10 @@ import static org.apache.hadoop.hbase.HConstants.NORMAL_QOS; import static org.apache.hadoop.hbase.HConstants.SYSTEMTABLE_QOS; import static org.apache.hadoop.hbase.NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; @@ -47,7 +47,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.ServerName; @@ -58,12 +57,10 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -88,13 +85,10 @@ /** * Confirm that we will set the priority in {@link HBaseRpcController} for several table operations. */ -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestAsyncTableRpcPriority { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncTableRpcPriority.class); - private static Configuration CONF = HBaseConfiguration.create(); private ClientService.Interface stub; @@ -103,11 +97,11 @@ public class TestAsyncTableRpcPriority { private AsyncConnection conn; - @Rule - public TestName name = new TestName(); + public String name; - @Before - public void setUp() throws IOException { + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException { + name = testInfo.getTestMethod().get().getName(); this.threadPool = Executors.newSingleThreadExecutor(); stub = mock(ClientService.Interface.class); @@ -217,21 +211,20 @@ public boolean matches(ScanRequest request) { @Test public void testGet() { - conn.getTable(TableName.valueOf(name.getMethodName())) - .get(new Get(Bytes.toBytes(0)).setPriority(11)).join(); + conn.getTable(TableName.valueOf(name)).get(new Get(Bytes.toBytes(0)).setPriority(11)).join(); verify(stub, times(1)).get(assertPriority(11), any(GetRequest.class), any()); } @Test public void testGetNormalTable() { - conn.getTable(TableName.valueOf(name.getMethodName())).get(new Get(Bytes.toBytes(0))).join(); + conn.getTable(TableName.valueOf(name)).get(new Get(Bytes.toBytes(0))).join(); verify(stub, times(1)).get(assertPriority(NORMAL_QOS), any(GetRequest.class), any()); } @Test public void testGetSystemTable() { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) - .get(new Get(Bytes.toBytes(0))).join(); + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)).get(new Get(Bytes.toBytes(0))) + .join(); verify(stub, times(1)).get(assertPriority(SYSTEMTABLE_QOS), any(GetRequest.class), any()); } @@ -243,26 +236,22 @@ public void testGetMetaTable() { @Test public void testPut() { - conn - .getTable(TableName.valueOf(name.getMethodName())).put(new Put(Bytes.toBytes(0)) - .setPriority(12).addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))) - .join(); + conn.getTable(TableName.valueOf(name)).put(new Put(Bytes.toBytes(0)).setPriority(12) + .addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))).join(); verify(stub, times(1)).mutate(assertPriority(12), any(MutateRequest.class), any()); } @Test public void testPutNormalTable() { - conn.getTable(TableName.valueOf(name.getMethodName())).put(new Put(Bytes.toBytes(0)) + conn.getTable(TableName.valueOf(name)).put(new Put(Bytes.toBytes(0)) .addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))).join(); verify(stub, times(1)).mutate(assertPriority(NORMAL_QOS), any(MutateRequest.class), any()); } @Test public void testPutSystemTable() { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) - .put(new Put(Bytes.toBytes(0)).addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), - Bytes.toBytes("v"))) - .join(); + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)).put(new Put(Bytes.toBytes(0)) + .addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))).join(); verify(stub, times(1)).mutate(assertPriority(SYSTEMTABLE_QOS), any(MutateRequest.class), any()); } @@ -275,21 +264,20 @@ public void testPutMetaTable() { @Test public void testDelete() { - conn.getTable(TableName.valueOf(name.getMethodName())) - .delete(new Delete(Bytes.toBytes(0)).setPriority(13)).join(); + conn.getTable(TableName.valueOf(name)).delete(new Delete(Bytes.toBytes(0)).setPriority(13)) + .join(); verify(stub, times(1)).mutate(assertPriority(13), any(MutateRequest.class), any()); } @Test public void testDeleteNormalTable() { - conn.getTable(TableName.valueOf(name.getMethodName())).delete(new Delete(Bytes.toBytes(0))) - .join(); + conn.getTable(TableName.valueOf(name)).delete(new Delete(Bytes.toBytes(0))).join(); verify(stub, times(1)).mutate(assertPriority(NORMAL_QOS), any(MutateRequest.class), any()); } @Test public void testDeleteSystemTable() { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)) .delete(new Delete(Bytes.toBytes(0))).join(); verify(stub, times(1)).mutate(assertPriority(SYSTEMTABLE_QOS), any(MutateRequest.class), any()); } @@ -302,23 +290,21 @@ public void testDeleteMetaTable() { @Test public void testAppend() { - conn - .getTable(TableName.valueOf(name.getMethodName())).append(new Append(Bytes.toBytes(0)) - .setPriority(14).addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))) - .join(); + conn.getTable(TableName.valueOf(name)).append(new Append(Bytes.toBytes(0)).setPriority(14) + .addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))).join(); verify(stub, times(1)).mutate(assertPriority(14), any(MutateRequest.class), any()); } @Test public void testAppendNormalTable() { - conn.getTable(TableName.valueOf(name.getMethodName())).append(new Append(Bytes.toBytes(0)) + conn.getTable(TableName.valueOf(name)).append(new Append(Bytes.toBytes(0)) .addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))).join(); verify(stub, times(1)).mutate(assertPriority(NORMAL_QOS), any(MutateRequest.class), any()); } @Test public void testAppendSystemTable() { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)) .append(new Append(Bytes.toBytes(0)).addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))) .join(); @@ -334,21 +320,21 @@ public void testAppendMetaTable() { @Test public void testIncrement() { - conn.getTable(TableName.valueOf(name.getMethodName())).increment(new Increment(Bytes.toBytes(0)) + conn.getTable(TableName.valueOf(name)).increment(new Increment(Bytes.toBytes(0)) .addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), 1).setPriority(15)).join(); verify(stub, times(1)).mutate(assertPriority(15), any(MutateRequest.class), any()); } @Test public void testIncrementNormalTable() { - conn.getTable(TableName.valueOf(name.getMethodName())) + conn.getTable(TableName.valueOf(name)) .incrementColumnValue(Bytes.toBytes(0), Bytes.toBytes("cf"), Bytes.toBytes("cq"), 1).join(); verify(stub, times(1)).mutate(assertPriority(NORMAL_QOS), any(MutateRequest.class), any()); } @Test public void testIncrementSystemTable() { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)) .incrementColumnValue(Bytes.toBytes(0), Bytes.toBytes("cf"), Bytes.toBytes("cq"), 1).join(); verify(stub, times(1)).mutate(assertPriority(SYSTEMTABLE_QOS), any(MutateRequest.class), any()); } @@ -362,9 +348,8 @@ public void testIncrementMetaTable() { @Test public void testCheckAndPut() { - conn.getTable(TableName.valueOf(name.getMethodName())) - .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) - .ifNotExists() + conn.getTable(TableName.valueOf(name)).checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")) + .qualifier(Bytes.toBytes("cq")).ifNotExists() .thenPut(new Put(Bytes.toBytes(0)) .addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v")).setPriority(16)) .join(); @@ -373,17 +358,16 @@ public void testCheckAndPut() { @Test public void testCheckAndPutNormalTable() { - conn.getTable(TableName.valueOf(name.getMethodName())) - .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) - .ifNotExists().thenPut(new Put(Bytes.toBytes(0)).addColumn(Bytes.toBytes("cf"), - Bytes.toBytes("cq"), Bytes.toBytes("v"))) + conn.getTable(TableName.valueOf(name)).checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")) + .qualifier(Bytes.toBytes("cq")).ifNotExists().thenPut(new Put(Bytes.toBytes(0)) + .addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))) .join(); verify(stub, times(1)).mutate(assertPriority(NORMAL_QOS), any(MutateRequest.class), any()); } @Test public void testCheckAndPutSystemTable() { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)) .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) .ifNotExists().thenPut(new Put(Bytes.toBytes(0)).addColumn(Bytes.toBytes("cf"), Bytes.toBytes("cq"), Bytes.toBytes("v"))) @@ -402,23 +386,23 @@ public void testCheckAndPutMetaTable() { @Test public void testCheckAndDelete() { - conn.getTable(TableName.valueOf(name.getMethodName())) - .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) - .ifEquals(Bytes.toBytes("v")).thenDelete(new Delete(Bytes.toBytes(0)).setPriority(17)).join(); + conn.getTable(TableName.valueOf(name)).checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")) + .qualifier(Bytes.toBytes("cq")).ifEquals(Bytes.toBytes("v")) + .thenDelete(new Delete(Bytes.toBytes(0)).setPriority(17)).join(); verify(stub, times(1)).mutate(assertPriority(17), any(MutateRequest.class), any()); } @Test public void testCheckAndDeleteNormalTable() { - conn.getTable(TableName.valueOf(name.getMethodName())) - .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) - .ifEquals(Bytes.toBytes("v")).thenDelete(new Delete(Bytes.toBytes(0))).join(); + conn.getTable(TableName.valueOf(name)).checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")) + .qualifier(Bytes.toBytes("cq")).ifEquals(Bytes.toBytes("v")) + .thenDelete(new Delete(Bytes.toBytes(0))).join(); verify(stub, times(1)).mutate(assertPriority(NORMAL_QOS), any(MutateRequest.class), any()); } @Test public void testCheckAndDeleteSystemTable() { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)) .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) .ifEquals(Bytes.toBytes("v")).thenDelete(new Delete(Bytes.toBytes(0))).join(); verify(stub, times(1)).mutate(assertPriority(SYSTEMTABLE_QOS), any(MutateRequest.class), any()); @@ -435,9 +419,9 @@ public void testCheckAndDeleteMetaTable() { @Test public void testCheckAndMutate() throws IOException { - conn.getTable(TableName.valueOf(name.getMethodName())) - .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) - .ifEquals(Bytes.toBytes("v")).thenMutate(new RowMutations(Bytes.toBytes(0)) + conn.getTable(TableName.valueOf(name)).checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")) + .qualifier(Bytes.toBytes("cq")).ifEquals(Bytes.toBytes("v")) + .thenMutate(new RowMutations(Bytes.toBytes(0)) .add((Mutation) new Delete(Bytes.toBytes(0)).setPriority(18))) .join(); verify(stub, times(1)).multi(assertPriority(18), any(ClientProtos.MultiRequest.class), any()); @@ -445,9 +429,8 @@ public void testCheckAndMutate() throws IOException { @Test public void testCheckAndMutateNormalTable() throws IOException { - conn.getTable(TableName.valueOf(name.getMethodName())) - .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) - .ifEquals(Bytes.toBytes("v")) + conn.getTable(TableName.valueOf(name)).checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")) + .qualifier(Bytes.toBytes("cq")).ifEquals(Bytes.toBytes("v")) .thenMutate(new RowMutations(Bytes.toBytes(0)).add((Mutation) new Delete(Bytes.toBytes(0)))) .join(); verify(stub, times(1)).multi(assertPriority(NORMAL_QOS), any(ClientProtos.MultiRequest.class), @@ -456,7 +439,7 @@ public void testCheckAndMutateNormalTable() throws IOException { @Test public void testCheckAndMutateSystemTable() throws IOException { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)) .checkAndMutate(Bytes.toBytes(0), Bytes.toBytes("cf")).qualifier(Bytes.toBytes("cq")) .ifEquals(Bytes.toBytes("v")) .thenMutate(new RowMutations(Bytes.toBytes(0)).add((Mutation) new Delete(Bytes.toBytes(0)))) @@ -495,8 +478,8 @@ public Void answer(InvocationOnMock invocation) throws Throwable { future.complete(null); } - assertFalse("close scanner should not come in with scan priority " + scanPriority, - req.hasCloseScanner() && req.getCloseScanner()); + assertFalse(req.hasCloseScanner() && req.getCloseScanner(), + "close scanner should not come in with scan priority " + scanPriority); Cell cell = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) .setType(Cell.Type.Put).setRow(Bytes.toBytes(scanNextCalled.incrementAndGet())) @@ -520,10 +503,10 @@ public Void answer(InvocationOnMock invocation) throws Throwable { threadPool.submit(() -> { ScanRequest req = invocation.getArgument(1); RpcCallback done = invocation.getArgument(2); - assertTrue("close request should have scannerId", req.hasScannerId()); - assertEquals("close request's scannerId should match", scannerId, req.getScannerId()); - assertTrue("close request should have closerScanner set", - req.hasCloseScanner() && req.getCloseScanner()); + assertTrue(req.hasScannerId(), "close request should have scannerId"); + assertEquals(scannerId, req.getScannerId(), "close request's scannerId should match"); + assertTrue(req.hasCloseScanner() && req.getCloseScanner(), + "close request should have closerScanner set"); done.run(ScanResponse.getDefaultInstance()); }); @@ -536,20 +519,19 @@ public Void answer(InvocationOnMock invocation) throws Throwable { @Test public void testScan() throws Exception { CompletableFuture renewFuture = mockScanReturnRenewFuture(19); - testForTable(TableName.valueOf(name.getMethodName()), renewFuture, Optional.of(19)); + testForTable(TableName.valueOf(name), renewFuture, Optional.of(19)); } @Test public void testScanNormalTable() throws Exception { CompletableFuture renewFuture = mockScanReturnRenewFuture(NORMAL_QOS); - testForTable(TableName.valueOf(name.getMethodName()), renewFuture, Optional.of(NORMAL_QOS)); + testForTable(TableName.valueOf(name), renewFuture, Optional.of(NORMAL_QOS)); } @Test public void testScanSystemTable() throws Exception { CompletableFuture renewFuture = mockScanReturnRenewFuture(SYSTEMTABLE_QOS); - testForTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName()), renewFuture, - Optional.empty()); + testForTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name), renewFuture, Optional.empty()); } @Test @@ -582,15 +564,15 @@ private void testForTable(TableName tableName, CompletableFuture renewFutu @Test public void testBatchNormalTable() { - conn.getTable(TableName.valueOf(name.getMethodName())) - .batchAll(Arrays.asList(new Delete(Bytes.toBytes(0)))).join(); + conn.getTable(TableName.valueOf(name)).batchAll(Arrays.asList(new Delete(Bytes.toBytes(0)))) + .join(); verify(stub, times(1)).multi(assertPriority(NORMAL_QOS), any(ClientProtos.MultiRequest.class), any()); } @Test public void testBatchSystemTable() { - conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName())) + conn.getTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name)) .batchAll(Arrays.asList(new Delete(Bytes.toBytes(0)))).join(); verify(stub, times(1)).multi(assertPriority(SYSTEMTABLE_QOS), any(ClientProtos.MultiRequest.class), any()); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableTracing.java index f7da22c45315..a19fdef59870 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableTracing.java @@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; @@ -41,7 +41,7 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; import io.opentelemetry.sdk.trace.data.SpanData; import java.io.IOException; import java.util.Arrays; @@ -56,7 +56,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MatcherPredicate; @@ -72,12 +71,11 @@ import org.apache.hadoop.hbase.util.Bytes; import org.hamcrest.Matcher; import org.hamcrest.core.IsAnything; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -99,13 +97,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestAsyncTableTracing { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncTableTracing.class); - private static Configuration CONF = HBaseConfiguration.create(); private ClientService.Interface stub; @@ -114,10 +109,10 @@ public class TestAsyncTableTracing { private AsyncTable table; - @Rule - public OpenTelemetryRule traceRule = OpenTelemetryRule.create(); + @RegisterExtension + public static final OpenTelemetryExtension traceRule = OpenTelemetryExtension.create(); - @Before + @BeforeEach public void setUp() throws IOException { stub = mock(ClientService.Interface.class); AtomicInteger scanNextCalled = new AtomicInteger(0); @@ -243,7 +238,7 @@ ClientService.Interface getRegionServerStub(ServerName serverName) throws IOExce table = conn.getTable(TableName.valueOf("table"), ForkJoinPool.commonPool()); } - @After + @AfterEach public void tearDown() throws IOException { Closeables.close(conn, true); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java index efb993561a5e..94ac18655fc8 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAttributes.java @@ -17,161 +17,154 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestAttributes { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAttributes.class); private static final byte[] ROW = new byte[] { 'r' }; @Test public void testPutAttributes() { Put put = new Put(ROW); - Assert.assertTrue(put.getAttributesMap().isEmpty()); - Assert.assertNull(put.getAttribute("absent")); + assertTrue(put.getAttributesMap().isEmpty()); + assertNull(put.getAttribute("absent")); put.setAttribute("absent", null); - Assert.assertTrue(put.getAttributesMap().isEmpty()); - Assert.assertNull(put.getAttribute("absent")); + assertTrue(put.getAttributesMap().isEmpty()); + assertNull(put.getAttribute("absent")); // adding attribute put.setAttribute("attribute1", Bytes.toBytes("value1")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), put.getAttribute("attribute1"))); - Assert.assertEquals(1, put.getAttributesMap().size()); - Assert - .assertTrue(Arrays.equals(Bytes.toBytes("value1"), put.getAttributesMap().get("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), put.getAttribute("attribute1"))); + assertEquals(1, put.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), put.getAttributesMap().get("attribute1"))); // overriding attribute value put.setAttribute("attribute1", Bytes.toBytes("value12")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), put.getAttribute("attribute1"))); - Assert.assertEquals(1, put.getAttributesMap().size()); - Assert.assertTrue( - Arrays.equals(Bytes.toBytes("value12"), put.getAttributesMap().get("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value12"), put.getAttribute("attribute1"))); + assertEquals(1, put.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value12"), put.getAttributesMap().get("attribute1"))); // adding another attribute put.setAttribute("attribute2", Bytes.toBytes("value2")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), put.getAttribute("attribute2"))); - Assert.assertEquals(2, put.getAttributesMap().size()); - Assert - .assertTrue(Arrays.equals(Bytes.toBytes("value2"), put.getAttributesMap().get("attribute2"))); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), put.getAttribute("attribute2"))); + assertEquals(2, put.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), put.getAttributesMap().get("attribute2"))); // removing attribute put.setAttribute("attribute2", null); - Assert.assertNull(put.getAttribute("attribute2")); - Assert.assertEquals(1, put.getAttributesMap().size()); - Assert.assertNull(put.getAttributesMap().get("attribute2")); + assertNull(put.getAttribute("attribute2")); + assertEquals(1, put.getAttributesMap().size()); + assertNull(put.getAttributesMap().get("attribute2")); // removing non-existed attribute put.setAttribute("attribute2", null); - Assert.assertNull(put.getAttribute("attribute2")); - Assert.assertEquals(1, put.getAttributesMap().size()); - Assert.assertNull(put.getAttributesMap().get("attribute2")); + assertNull(put.getAttribute("attribute2")); + assertEquals(1, put.getAttributesMap().size()); + assertNull(put.getAttributesMap().get("attribute2")); // removing another attribute put.setAttribute("attribute1", null); - Assert.assertNull(put.getAttribute("attribute1")); - Assert.assertTrue(put.getAttributesMap().isEmpty()); - Assert.assertNull(put.getAttributesMap().get("attribute1")); + assertNull(put.getAttribute("attribute1")); + assertTrue(put.getAttributesMap().isEmpty()); + assertNull(put.getAttributesMap().get("attribute1")); } @Test public void testDeleteAttributes() { Delete del = new Delete(new byte[] { 'r' }); - Assert.assertTrue(del.getAttributesMap().isEmpty()); - Assert.assertNull(del.getAttribute("absent")); + assertTrue(del.getAttributesMap().isEmpty()); + assertNull(del.getAttribute("absent")); del.setAttribute("absent", null); - Assert.assertTrue(del.getAttributesMap().isEmpty()); - Assert.assertNull(del.getAttribute("absent")); + assertTrue(del.getAttributesMap().isEmpty()); + assertNull(del.getAttribute("absent")); // adding attribute del.setAttribute("attribute1", Bytes.toBytes("value1")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), del.getAttribute("attribute1"))); - Assert.assertEquals(1, del.getAttributesMap().size()); - Assert - .assertTrue(Arrays.equals(Bytes.toBytes("value1"), del.getAttributesMap().get("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), del.getAttribute("attribute1"))); + assertEquals(1, del.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), del.getAttributesMap().get("attribute1"))); // overriding attribute value del.setAttribute("attribute1", Bytes.toBytes("value12")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), del.getAttribute("attribute1"))); - Assert.assertEquals(1, del.getAttributesMap().size()); - Assert.assertTrue( - Arrays.equals(Bytes.toBytes("value12"), del.getAttributesMap().get("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value12"), del.getAttribute("attribute1"))); + assertEquals(1, del.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value12"), del.getAttributesMap().get("attribute1"))); // adding another attribute del.setAttribute("attribute2", Bytes.toBytes("value2")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), del.getAttribute("attribute2"))); - Assert.assertEquals(2, del.getAttributesMap().size()); - Assert - .assertTrue(Arrays.equals(Bytes.toBytes("value2"), del.getAttributesMap().get("attribute2"))); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), del.getAttribute("attribute2"))); + assertEquals(2, del.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), del.getAttributesMap().get("attribute2"))); // removing attribute del.setAttribute("attribute2", null); - Assert.assertNull(del.getAttribute("attribute2")); - Assert.assertEquals(1, del.getAttributesMap().size()); - Assert.assertNull(del.getAttributesMap().get("attribute2")); + assertNull(del.getAttribute("attribute2")); + assertEquals(1, del.getAttributesMap().size()); + assertNull(del.getAttributesMap().get("attribute2")); // removing non-existed attribute del.setAttribute("attribute2", null); - Assert.assertNull(del.getAttribute("attribute2")); - Assert.assertEquals(1, del.getAttributesMap().size()); - Assert.assertNull(del.getAttributesMap().get("attribute2")); + assertNull(del.getAttribute("attribute2")); + assertEquals(1, del.getAttributesMap().size()); + assertNull(del.getAttributesMap().get("attribute2")); // removing another attribute del.setAttribute("attribute1", null); - Assert.assertNull(del.getAttribute("attribute1")); - Assert.assertTrue(del.getAttributesMap().isEmpty()); - Assert.assertNull(del.getAttributesMap().get("attribute1")); + assertNull(del.getAttribute("attribute1")); + assertTrue(del.getAttributesMap().isEmpty()); + assertNull(del.getAttributesMap().get("attribute1")); } @Test public void testGetId() { Get get = new Get(ROW); - Assert.assertNull("Make sure id is null if unset", get.toMap().get("id")); + assertNull(get.toMap().get("id"), "Make sure id is null if unset"); get.setId("myId"); - Assert.assertEquals("myId", get.toMap().get("id")); + assertEquals("myId", get.toMap().get("id")); } @Test public void testAppendId() { Append append = new Append(ROW); - Assert.assertNull("Make sure id is null if unset", append.toMap().get("id")); + assertNull(append.toMap().get("id"), "Make sure id is null if unset"); append.setId("myId"); - Assert.assertEquals("myId", append.toMap().get("id")); + assertEquals("myId", append.toMap().get("id")); } @Test public void testDeleteId() { Delete delete = new Delete(ROW); - Assert.assertNull("Make sure id is null if unset", delete.toMap().get("id")); + assertNull(delete.toMap().get("id"), "Make sure id is null if unset"); delete.setId("myId"); - Assert.assertEquals("myId", delete.toMap().get("id")); + assertEquals("myId", delete.toMap().get("id")); } @Test public void testPutId() { Put put = new Put(ROW); - Assert.assertNull("Make sure id is null if unset", put.toMap().get("id")); + assertNull(put.toMap().get("id"), "Make sure id is null if unset"); put.setId("myId"); - Assert.assertEquals("myId", put.toMap().get("id")); + assertEquals("myId", put.toMap().get("id")); } @Test public void testScanId() { Scan scan = new Scan(); - Assert.assertNull("Make sure id is null if unset", scan.toMap().get("id")); + assertNull(scan.toMap().get("id"), "Make sure id is null if unset"); scan.setId("myId"); - Assert.assertEquals("myId", scan.toMap().get("id")); + assertEquals("myId", scan.toMap().get("id")); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestBufferedMutator.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestBufferedMutator.java index 2d83c29bae99..ec69a4004697 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestBufferedMutator.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestBufferedMutator.java @@ -17,33 +17,24 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -@Category({ SmallTests.class, ClientTests.class }) +@Tag(SmallTests.TAG) +@Tag(ClientTests.TAG) public class TestBufferedMutator { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBufferedMutator.class); - - @Rule - public TestName name = new TestName(); - /** * My BufferedMutator. Just to prove that I can insert a BM other than default. */ @@ -55,9 +46,9 @@ public static class MyBufferedMutator extends BufferedMutatorImpl { } @Test - public void testAlternateBufferedMutatorImpl() throws IOException { + public void testAlternateBufferedMutatorImpl(TestInfo testInfo) throws IOException { BufferedMutatorParams params = - new BufferedMutatorParams(TableName.valueOf(name.getMethodName())); + new BufferedMutatorParams(TableName.valueOf(testInfo.getTestMethod().get().getName())); Configuration conf = HBaseConfiguration.create(); conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, DoNothingConnectionRegistry.class.getName()); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestBufferedMutatorParams.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestBufferedMutatorParams.java index b6c52a0cd0d6..d6e44b5c1ebb 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestBufferedMutatorParams.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestBufferedMutatorParams.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.util.Collection; @@ -30,26 +30,17 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestBufferedMutatorParams { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBufferedMutatorParams.class); - - @Rule - public TestName name = new TestName(); - /** * Just to create in instance, this doesn't actually function. */ @@ -134,9 +125,10 @@ public void onException(RetriesExhaustedWithDetailsException exception, Buffered } @Test - public void testClone() { + public void testClone(TestInfo testInfo) { ExecutorService pool = new MockExecutorService(); - final String tableName = name.getMethodName(); + final String tableName = testInfo.getTestMethod().get().getName(); + ; BufferedMutatorParams bmp = new BufferedMutatorParams(TableName.valueOf(tableName)); BufferedMutator.ExceptionListener listener = new MockExceptionListener(); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java index 6ae602aaadb5..c1045c8ed50a 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java @@ -17,30 +17,26 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.backoff.ExponentialClientBackoffPolicy; import org.apache.hadoop.hbase.client.backoff.ServerStatistics; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestClientExponentialBackoff { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestClientExponentialBackoff.class); ServerName server = Mockito.mock(ServerName.class); byte[] regionname = Bytes.toBytes("region"); @@ -105,8 +101,8 @@ public void testResultOrdering() { for (int i = 1; i <= 100; i++) { update(stats, i); long next = backoff.getBackoffTime(server, regionname, stats); - assertTrue("Previous backoff time" + previous + " >= " + next + ", the next backoff time for " - + "load " + i, previous < next); + assertTrue(previous < next, "Previous backoff time" + previous + " >= " + next + + ", the next backoff time for " + "load " + i); previous = next; } } @@ -121,18 +117,18 @@ public void testHeapOccupancyPolicy() { update(stats, 0, 95, 0); backoffTime = backoff.getBackoffTime(server, regionname, stats); - assertTrue("Heap occupancy at low watermark had no effect", backoffTime > 0); + assertTrue(backoffTime > 0, "Heap occupancy at low watermark had no effect"); long previous = backoffTime; update(stats, 0, 96, 0); backoffTime = backoff.getBackoffTime(server, regionname, stats); - assertTrue("Increase above low watermark should have increased backoff", - backoffTime > previous); + assertTrue(backoffTime > previous, + "Increase above low watermark should have increased backoff"); update(stats, 0, 98, 0); backoffTime = backoff.getBackoffTime(server, regionname, stats); - assertEquals("We should be using max backoff when at high watermark", - ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoffTime); + assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoffTime, + "We should be using max backoff when at high watermark"); } @Test @@ -145,17 +141,17 @@ public void testCompactionPressurePolicy() { update(stats, 0, 0, 0); backoffTime = backoff.getBackoffTime(server, regionname, stats); - assertTrue("Compaction pressure has no effect", backoffTime == 0); + assertTrue(backoffTime == 0, "Compaction pressure has no effect"); long previous = backoffTime; update(stats, 0, 0, 50); backoffTime = backoff.getBackoffTime(server, regionname, stats); - assertTrue("Compaction pressure should be bigger", backoffTime > previous); + assertTrue(backoffTime > previous, "Compaction pressure should be bigger"); update(stats, 0, 0, 100); backoffTime = backoff.getBackoffTime(server, regionname, stats); - assertEquals("under heavy compaction pressure", - ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoffTime); + assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoffTime, + "under heavy compaction pressure"); } private void update(ServerStatistics stats, int load) { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java index a094004f6a70..593d655a7a33 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.net.SocketTimeoutException; @@ -39,7 +39,6 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; @@ -61,11 +60,10 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,13 +103,10 @@ * Test client behavior w/o setting up a cluster. Mock up cluster emissions. See below for a method * that tests retries/timeouts currently commented out. */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestClientNoCluster extends Configured implements Tool { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestClientNoCluster.class); - private static final Logger LOG = LoggerFactory.getLogger(TestClientNoCluster.class); private Configuration conf; /** @@ -123,7 +118,7 @@ public class TestClientNoCluster extends Configured implements Tool { public static final ServerName META_SERVERNAME = ServerName.valueOf("meta.example.org", 16010, 12345); - @Before + @BeforeEach public void setUp() throws Exception { this.conf = HBaseConfiguration.create(); // Run my Connection overrides. Use my little ConnectionImplementation below which @@ -157,7 +152,7 @@ public CompletableFuture getClusterId() { /** * Remove the @Ignore to try out timeout and retry settings */ - @Ignore + @Disabled @Test public void testTimeoutAndRetries() throws IOException { Configuration localConfig = HBaseConfiguration.create(this.conf); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientScanner.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientScanner.java index e039be8baa83..bc977601eb14 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientScanner.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientScanner.java @@ -19,11 +19,11 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.RegionLocations; @@ -46,28 +45,19 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.InOrder; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -/** - * Test the ClientScanner. - */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestClientScanner { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestClientScanner.class); - Scan scan; ExecutorService pool; Configuration conf; @@ -77,11 +67,11 @@ public class TestClientScanner { RpcRetryingCallerFactory rpcFactory; RpcControllerFactory controllerFactory; - @Rule - public TestName name = new TestName(); + private String methodName; - @Before - public void setup() throws IOException { + @BeforeEach + public void setup(TestInfo testInfo) throws IOException { + methodName = testInfo.getTestMethod().get().getName(); clusterConn = Mockito.mock(ClusterConnection.class); rpcFactory = Mockito.mock(RpcRetryingCallerFactory.class); controllerFactory = Mockito.mock(RpcControllerFactory.class); @@ -93,7 +83,7 @@ public void setup() throws IOException { Mockito.when(clusterConn.getConnectionConfiguration()).thenReturn(connectionConfig); } - @After + @AfterEach public void teardown() { if (null != pool) { pool.shutdownNow(); @@ -176,9 +166,9 @@ public Result[] answer(InvocationOnMock invocation) throws Throwable { scan.setCaching(100); scan.setMaxResultSize(1000 * 1000); - try (MockClientScanner scanner = - new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, - rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { + try ( + MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(methodName), + clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { scanner.setRpcFinished(true); @@ -240,9 +230,9 @@ public Result[] answer(InvocationOnMock invocation) throws Throwable { // The single key-value will exit the loop scan.setMaxResultSize(1); - try (MockClientScanner scanner = - new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, - rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { + try ( + MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(methodName), + clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { InOrder inOrder = Mockito.inOrder(caller); scanner.loadCache(); @@ -303,9 +293,9 @@ public Result[] answer(InvocationOnMock invocation) throws Throwable { // Set a very large size scan.setMaxResultSize(1000 * 1000); - try (MockClientScanner scanner = - new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, - rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { + try ( + MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(methodName), + clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { InOrder inOrder = Mockito.inOrder(caller); scanner.loadCache(); @@ -374,9 +364,9 @@ public Result[] answer(InvocationOnMock invocation) throws Throwable { scan.setCaching(100); scan.setMaxResultSize(1000 * 1000); - try (MockClientScanner scanner = - new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, - rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { + try ( + MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(methodName), + clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { scanner.setRpcFinished(true); InOrder inOrder = Mockito.inOrder(caller); @@ -441,9 +431,9 @@ public Result[] answer(InvocationOnMock invocation) throws Throwable { scan.setCaching(100); scan.setMaxResultSize(1000 * 1000); - try (MockClientScanner scanner = - new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, - rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { + try ( + MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(methodName), + clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE, connectionConfig)) { InOrder inOrder = Mockito.inOrder(caller); scanner.setRpcFinished(true); @@ -487,8 +477,8 @@ public void testExceptionsFromReplicasArePropagated() throws IOException { anyInt())).thenReturn(new RegionLocations(null, null, null)); try (MockClientScanner scanner = - new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, - rpcFactory, new RpcControllerFactory(conf), pool, Integer.MAX_VALUE, connectionConfig)) { + new MockClientScanner(conf, scan, TableName.valueOf(methodName), clusterConn, rpcFactory, + new RpcControllerFactory(conf), pool, Integer.MAX_VALUE, connectionConfig)) { Iterator iter = scanner.iterator(); while (iter.hasNext()) { iter.next(); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorBuilder.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorBuilder.java index 7a6f8e92226a..9e23e9d2f056 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorBuilder.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorBuilder.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Map; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeepDeletedCells; @@ -38,21 +38,12 @@ import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.PrettyPrinter; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestColumnFamilyDescriptorBuilder { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestColumnFamilyDescriptorBuilder.class); - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); @Test public void testBuilder() throws DeserializationException { @@ -105,9 +96,8 @@ public void testBuilder() throws DeserializationException { */ @Test public void testHColumnDescriptorShouldThrowIAEWhenFamilyNameEmpty() throws Exception { - expectedEx.expect(IllegalArgumentException.class); - expectedEx.expectMessage("Column Family name can not be empty"); - ColumnFamilyDescriptorBuilder.of(""); + assertThrows(IllegalArgumentException.class, () -> ColumnFamilyDescriptorBuilder.of(""), + "Column Family name can not be empty"); } /** @@ -163,31 +153,31 @@ public void testSetTimeToLive() throws HBaseException { ttl = "50000"; builder.setTimeToLive(ttl); - Assert.assertEquals(50000, builder.build().getTimeToLive()); + assertEquals(50000, builder.build().getTimeToLive()); ttl = "50000 seconds"; builder.setTimeToLive(ttl); - Assert.assertEquals(50000, builder.build().getTimeToLive()); + assertEquals(50000, builder.build().getTimeToLive()); ttl = ""; builder.setTimeToLive(ttl); - Assert.assertEquals(0, builder.build().getTimeToLive()); + assertEquals(0, builder.build().getTimeToLive()); ttl = "FOREVER"; builder.setTimeToLive(ttl); - Assert.assertEquals(HConstants.FOREVER, builder.build().getTimeToLive()); + assertEquals(HConstants.FOREVER, builder.build().getTimeToLive()); ttl = "1 HOUR 10 minutes 1 second"; builder.setTimeToLive(ttl); - Assert.assertEquals(4201, builder.build().getTimeToLive()); + assertEquals(4201, builder.build().getTimeToLive()); ttl = "500 Days 23 HOURS"; builder.setTimeToLive(ttl); - Assert.assertEquals(43282800, builder.build().getTimeToLive()); + assertEquals(43282800, builder.build().getTimeToLive()); ttl = "43282800 SECONDS (500 Days 23 hours)"; builder.setTimeToLive(ttl); - Assert.assertEquals(43282800, builder.build().getTimeToLive()); + assertEquals(43282800, builder.build().getTimeToLive()); } @Test diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorLowerCaseEnum.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorLowerCaseEnum.java index 900f4093d1b0..277b677eb447 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorLowerCaseEnum.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestColumnFamilyDescriptorLowerCaseEnum.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -25,26 +25,21 @@ import java.util.HashMap; import java.util.Map; import java.util.NoSuchElementException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Testcase for HBASE-21732. Make sure that all enum configurations can accept lower case value. */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestColumnFamilyDescriptorLowerCaseEnum { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestColumnFamilyDescriptorLowerCaseEnum.class); - private static final Logger LOG = LoggerFactory.getLogger(TestColumnFamilyDescriptorLowerCaseEnum.class); @@ -97,8 +92,8 @@ public void test() ColumnFamilyDescriptor desc2 = builder2.build(); for (Map.Entry> entry : getMethod2Value.entrySet()) { - assertEquals(entry.getKey() + " should return " + entry.getValue(), entry.getValue(), - entry.getKey().invoke(desc2)); + assertEquals(entry.getValue(), entry.getKey().invoke(desc2), + entry.getKey() + " should return " + entry.getValue()); } } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestCompactFromClient.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestCompactFromClient.java new file mode 100644 index 000000000000..694c7f5d5577 --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestCompactFromClient.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hadoop.hbase.AsyncMetaTableAccessor; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.FutureUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer; + +@Tag(MediumTests.TAG) +@Tag(ClientTests.TAG) +public class TestCompactFromClient { + + private org.apache.logging.log4j.core.Appender mockAppender; + + @BeforeEach + public void setUp() { + mockAppender = mock(org.apache.logging.log4j.core.Appender.class); + when(mockAppender.getName()).thenReturn("mockAppender"); + when(mockAppender.isStarted()).thenReturn(true); + } + + @AfterEach + public void tearDown() { + ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager + .getLogger(FutureUtils.class)).removeAppender(mockAppender); + } + + @Test + public void testCompactTableWithNullLocations() throws Exception { + AtomicReference msg = new AtomicReference<>(); + AtomicReference throwable = new AtomicReference<>(); + + ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager + .getLogger(FutureUtils.class)).addAppender(mockAppender); + doAnswer(invocation -> { + org.apache.logging.log4j.core.LogEvent logEvent = + invocation.getArgument(0, org.apache.logging.log4j.core.LogEvent.class); + org.apache.logging.log4j.Level originalLevel = logEvent.getLevel(); + if (originalLevel.equals(org.apache.logging.log4j.Level.ERROR)) { + msg.set(logEvent.getMessage().getFormattedMessage()); + throwable.set(logEvent.getThrown()); + } + return null; + }).when(mockAppender).append(any(org.apache.logging.log4j.core.LogEvent.class)); + + TableName tableName = TableName.valueOf("testCompactNullLocations"); + CompletableFuture> nullLocationsFuture = + CompletableFuture.completedFuture(null); + + try (MockedStatic mockedMeta = mockStatic(AsyncMetaTableAccessor.class); + AsyncConnectionImpl connection = mock(AsyncConnectionImpl.class)) { + @SuppressWarnings("unchecked") + AsyncTable table = any(AsyncTable.class); + mockedMeta + .when(() -> AsyncMetaTableAccessor.getTableHRegionLocations(table, any(TableName.class))) + .thenReturn(nullLocationsFuture); + @SuppressWarnings("unchecked") + AsyncTable metaTable = mock(AsyncTable.class); + when(connection.getTable(META_TABLE_NAME)).thenReturn(metaTable); + + HashedWheelTimer hashedWheelTimer = mock(HashedWheelTimer.class); + AsyncAdminBuilderBase asyncAdminBuilderBase = mock(AsyncAdminBuilderBase.class); + RawAsyncHBaseAdmin admin = + new RawAsyncHBaseAdmin(connection, hashedWheelTimer, asyncAdminBuilderBase); + + CompletableFuture future = admin.compact(tableName, CompactType.NORMAL); + // future.get() throws ExecutionException, and the cause is TableNotFoundException + ExecutionException ex = + assertThrows(ExecutionException.class, () -> future.get(1, TimeUnit.SECONDS)); + assertInstanceOf(TableNotFoundException.class, ex.getCause(), + "Expected TableNotFoundException as the cause of ExecutionException"); + + assertNull(msg.get(), "No error message should be logged"); + assertNull(throwable.get(), "No Exception should be logged"); + } + } +} diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionConfiguration.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionConfiguration.java index d3031fe85c79..66375000049e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionConfiguration.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionConfiguration.java @@ -17,25 +17,20 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestConnectionConfiguration { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestConnectionConfiguration.class); - @Test public void itHandlesDeprecatedPauseForCQTBE() { Configuration conf = new Configuration(); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionFactoryTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionFactoryTracing.java index 0e61dfc9db21..dbbda77d2e8c 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionFactoryTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionFactoryTracing.java @@ -22,38 +22,33 @@ import java.io.Closeable; import java.io.IOException; import java.util.concurrent.ExecutionException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestConnectionFactoryTracing extends TestTracingBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestConnectionFactoryTracing.class); - private User currentUser; private Object connection; @Override - @Before + @BeforeEach public void setUp() throws Exception { super.setUp(); currentUser = UserProvider.instantiate(conf).getCurrent(); conf.set(CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, RegistryForTracingTest.class.getName()); } - @After + @AfterEach public void tearDown() throws IOException { Closeables.close((Closeable) connection, true); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementationTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementationTracing.java index a207186785a8..a1f91d583dbc 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementationTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementationTracing.java @@ -19,37 +19,32 @@ import java.io.IOException; import java.util.Collections; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestConnectionImplementationTracing extends TestTracingBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestConnectionImplementationTracing.class); - ConnectionImplementation conn; @Override - @Before + @BeforeEach public void setUp() throws Exception { super.setUp(); conn = new ConnectionImplementation(conf, null, UserProvider.instantiate(conf).getCurrent(), Collections.emptyMap()); } - @After + @AfterEach public void tearDown() throws IOException { Closeables.close(conn, true); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionRegistryLeak.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionRegistryLeak.java index bd2ca9867f34..b026fac149da 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionRegistryLeak.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionRegistryLeak.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.ArrayList; @@ -27,25 +27,20 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FutureUtils; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestConnectionRegistryLeak { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestConnectionRegistryLeak.class); - public static final class ConnectionRegistryForTest extends DoNothingConnectionRegistry { private boolean closed = false; @@ -70,7 +65,7 @@ public void close() { private static Configuration CONF = HBaseConfiguration.create(); - @BeforeClass + @BeforeAll public static void setUp() { CONF.setClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ConnectionRegistryForTest.class, ConnectionRegistry.class); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestCoprocessorDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestCoprocessorDescriptor.java index a56c863ce614..108985f6be8b 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestCoprocessorDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestCoprocessorDescriptor.java @@ -17,33 +17,24 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestCoprocessorDescriptor { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCoprocessorDescriptor.class); - - @Rule - public TestName name = new TestName(); - @Test public void testBuild() { String className = "className"; @@ -61,7 +52,8 @@ public void testBuild() { } @Test - public void testSetCoprocessor() throws IOException { + public void testSetCoprocessor(TestInfo testInfo) throws IOException { + String name = testInfo.getTestMethod().get().getName(); String propertyKey = "propertyKey"; List cps = new ArrayList<>(); for (String className : Arrays.asList("className0", "className1", "className2")) { @@ -71,8 +63,8 @@ public void testSetCoprocessor() throws IOException { cps.add(CoprocessorDescriptorBuilder.newBuilder(className).setJarPath(path) .setPriority(priority).setProperty(propertyKey, propertyValue).build()); } - TableDescriptor tableDescriptor = TableDescriptorBuilder - .newBuilder(TableName.valueOf(name.getMethodName())).setCoprocessors(cps).build(); + TableDescriptor tableDescriptor = + TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setCoprocessors(cps).build(); for (CoprocessorDescriptor cp : cps) { boolean match = false; for (CoprocessorDescriptor that : tableDescriptor.getCoprocessorDescriptors()) { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestDelayingRunner.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestDelayingRunner.java index 30d7c2cb2eb8..1a02b4de0004 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestDelayingRunner.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestDelayingRunner.java @@ -17,27 +17,21 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestDelayingRunner { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestDelayingRunner.class); - private static final TableName DUMMY_TABLE = TableName.valueOf("DUMMY_TABLE"); private static final byte[] DUMMY_BYTES_1 = Bytes.toBytes("DUMMY_BYTES_1"); private static final byte[] DUMMY_BYTES_2 = Bytes.toBytes("DUMMY_BYTES_2"); @@ -61,8 +55,8 @@ public void run() { long startTime = EnvironmentEdgeManager.currentTime(); runner.run(); long delay = endTime.get() - startTime; - assertTrue("DelayingRunner did not delay long enough", delay >= sleepTime); - assertFalse("DelayingRunner delayed too long", delay > sleepTime + sleepTime * 0.2); + assertTrue(delay >= sleepTime, "DelayingRunner did not delay long enough"); + assertFalse(delay > sleepTime + sleepTime * 0.2, "DelayingRunner delayed too long"); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestDeleteTimeStamp.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestDeleteTimeStamp.java index cce3ba4e4e3f..0818ee1057d4 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestDeleteTimeStamp.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestDeleteTimeStamp.java @@ -17,26 +17,22 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.List; import java.util.Map.Entry; import java.util.NavigableMap; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestDeleteTimeStamp { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestDeleteTimeStamp.class); - private static final byte[] ROW = Bytes.toBytes("testRow"); private static final byte[] FAMILY = Bytes.toBytes("testFamily"); private static final byte[] QUALIFIER = Bytes.toBytes("testQualifier"); @@ -53,7 +49,7 @@ public void testTimeStamp() { NavigableMap> familyCellmap = delete.getFamilyCellMap(); for (Entry> entry : familyCellmap.entrySet()) { for (Cell cell : entry.getValue()) { - Assert.assertEquals(ts, cell.getTimestamp()); + assertEquals(ts, cell.getTimestamp()); } } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java index 3815951d4e33..b9f0de4d3675 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java @@ -19,10 +19,12 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.File; import java.io.FileOutputStream; @@ -33,7 +35,6 @@ import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; @@ -43,10 +44,8 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.base.Throwables; @@ -54,10 +53,9 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; // TODO: cover more test cases -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestGet { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestGet.class); private static final byte[] ROW = new byte[] { 'r' }; @@ -101,61 +99,58 @@ public void testAttributesSerialization() throws IOException { ClientProtos.Get getProto = ProtobufUtil.toGet(get); Get get2 = ProtobufUtil.toGet(getProto); - Assert.assertNull(get2.getAttribute("absent")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), get2.getAttribute("attribute1"))); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), get2.getAttribute("attribute2"))); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), get2.getAttribute("attribute3"))); - Assert.assertEquals(3, get2.getAttributesMap().size()); + assertNull(get2.getAttribute("absent")); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), get2.getAttribute("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), get2.getAttribute("attribute2"))); + assertTrue(Arrays.equals(Bytes.toBytes("value3"), get2.getAttribute("attribute3"))); + assertEquals(3, get2.getAttributesMap().size()); } @Test public void testGetAttributes() { Get get = new Get(ROW); - Assert.assertTrue(get.getAttributesMap().isEmpty()); - Assert.assertNull(get.getAttribute("absent")); + assertTrue(get.getAttributesMap().isEmpty()); + assertNull(get.getAttribute("absent")); get.setAttribute("absent", null); - Assert.assertTrue(get.getAttributesMap().isEmpty()); - Assert.assertNull(get.getAttribute("absent")); + assertTrue(get.getAttributesMap().isEmpty()); + assertNull(get.getAttribute("absent")); // adding attribute get.setAttribute("attribute1", Bytes.toBytes("value1")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), get.getAttribute("attribute1"))); - Assert.assertEquals(1, get.getAttributesMap().size()); - Assert - .assertTrue(Arrays.equals(Bytes.toBytes("value1"), get.getAttributesMap().get("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), get.getAttribute("attribute1"))); + assertEquals(1, get.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), get.getAttributesMap().get("attribute1"))); // overriding attribute value get.setAttribute("attribute1", Bytes.toBytes("value12")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), get.getAttribute("attribute1"))); - Assert.assertEquals(1, get.getAttributesMap().size()); - Assert.assertTrue( - Arrays.equals(Bytes.toBytes("value12"), get.getAttributesMap().get("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value12"), get.getAttribute("attribute1"))); + assertEquals(1, get.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value12"), get.getAttributesMap().get("attribute1"))); // adding another attribute get.setAttribute("attribute2", Bytes.toBytes("value2")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), get.getAttribute("attribute2"))); - Assert.assertEquals(2, get.getAttributesMap().size()); - Assert - .assertTrue(Arrays.equals(Bytes.toBytes("value2"), get.getAttributesMap().get("attribute2"))); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), get.getAttribute("attribute2"))); + assertEquals(2, get.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), get.getAttributesMap().get("attribute2"))); // removing attribute get.setAttribute("attribute2", null); - Assert.assertNull(get.getAttribute("attribute2")); - Assert.assertEquals(1, get.getAttributesMap().size()); - Assert.assertNull(get.getAttributesMap().get("attribute2")); + assertNull(get.getAttribute("attribute2")); + assertEquals(1, get.getAttributesMap().size()); + assertNull(get.getAttributesMap().get("attribute2")); // removing non-existed attribute get.setAttribute("attribute2", null); - Assert.assertNull(get.getAttribute("attribute2")); - Assert.assertEquals(1, get.getAttributesMap().size()); - Assert.assertNull(get.getAttributesMap().get("attribute2")); + assertNull(get.getAttribute("attribute2")); + assertEquals(1, get.getAttributesMap().size()); + assertNull(get.getAttributesMap().get("attribute2")); // removing another attribute get.setAttribute("attribute1", null); - Assert.assertNull(get.getAttribute("attribute1")); - Assert.assertTrue(get.getAttributesMap().isEmpty()); - Assert.assertNull(get.getAttributesMap().get("attribute1")); + assertNull(get.getAttribute("attribute1")); + assertTrue(get.getAttributesMap().isEmpty()); + assertNull(get.getAttributesMap().get("attribute1")); } @Test @@ -164,7 +159,7 @@ public void testNullQualifier() { byte[] family = Bytes.toBytes("family"); get.addColumn(family, null); Set qualifiers = get.getFamilyMap().get(family); - Assert.assertEquals(1, qualifiers.size()); + assertEquals(1, qualifiers.size()); } @Test @@ -215,7 +210,7 @@ public void testDynamicFilter() throws Exception { String localPath = conf.get("hbase.local.dir") + File.separator + "jars" + File.separator; File jarFile = new File(localPath, "MockFilter.jar"); jarFile.delete(); - assertFalse("Should be deleted: " + jarFile.getPath(), jarFile.exists()); + assertFalse(jarFile.exists(), "Should be deleted: " + jarFile.getPath()); ClientProtos.Get getProto1 = ClientProtos.Get.parseFrom(Base64.getDecoder().decode(PB_GET)); ClientProtos.Get getProto2 = @@ -257,6 +252,6 @@ public void testGetRowConstructor() { ByteBuffer rowBuffer = ByteBuffer.wrap(row1); Get get1 = new Get(rowBuffer); Get get2 = new Get(row2, 4, 7); - Assert.assertArrayEquals(get1.getRow(), get2.getRow()); + assertArrayEquals(get1.getRow(), get2.getRow()); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexerViaMocks.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexerViaMocks.java index 22b8576611fd..3ca5164c1567 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexerViaMocks.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexerViaMocks.java @@ -26,25 +26,19 @@ import static org.mockito.Mockito.when; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestHTableMultiplexerViaMocks { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHTableMultiplexerViaMocks.class); - private HTableMultiplexer mockMultiplexer; private ClusterConnection mockConnection; - @Before + @BeforeEach public void setupTest() { mockMultiplexer = mock(HTableMultiplexer.class); mockConnection = mock(ClusterConnection.class); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestHTableTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestHTableTracing.java index 3a0f54a2af1d..68a793f29b29 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestHTableTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestHTableTracing.java @@ -52,7 +52,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MatcherPredicate; import org.apache.hadoop.hbase.ServerName; @@ -65,11 +64,10 @@ import org.apache.hadoop.hbase.util.Bytes; import org.hamcrest.Matcher; import org.hamcrest.core.IsAnything; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -92,18 +90,16 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestHTableTracing extends TestTracingBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHTableTracing.class); private ClientProtos.ClientService.BlockingInterface stub; private ConnectionImplementation conn; private Table table; @Override - @Before + @BeforeEach public void setUp() throws Exception { super.setUp(); @@ -230,7 +226,7 @@ public ClientService.BlockingInterface getClient(ServerName serverName) throws I table = conn.getTable(TableName.META_TABLE_NAME, ForkJoinPool.commonPool()); } - @After + @AfterEach public void tearDown() throws IOException { Closeables.close(conn, true); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHColumnDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHColumnDescriptor.java index f7c2e3b52263..1a783e0d7292 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHColumnDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHColumnDescriptor.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; +import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; import java.util.function.Consumer; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.MemoryCompactionPolicy; @@ -33,21 +33,14 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestImmutableHColumnDescriptor { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestImmutableHColumnDescriptor.class); - - @Rule - public TestName name = new TestName(); private static final List> TEST_FUNCTION = Arrays.asList( hcd -> hcd.setValue("a", "a"), hcd -> hcd.setValue(Bytes.toBytes("a"), Bytes.toBytes("a")), hcd -> hcd.setConfiguration("aaa", "ccc"), hcd -> hcd.remove(Bytes.toBytes("aaa")), @@ -71,9 +64,10 @@ public class TestImmutableHColumnDescriptor { hcd -> hcd.setVersions(1, 10)); @Test - public void testImmutable() { + public void testImmutable(TestInfo testInfo) { + String methodName = testInfo.getTestMethod().map(Method::getName).orElse("default"); ImmutableHColumnDescriptor hcd = - new ImmutableHColumnDescriptor(new HColumnDescriptor(Bytes.toBytes(name.getMethodName()))); + new ImmutableHColumnDescriptor(new HColumnDescriptor(Bytes.toBytes(methodName))); for (int i = 0; i != TEST_FUNCTION.size(); ++i) { try { TEST_FUNCTION.get(i).accept(hcd); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHRegionInfo.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHRegionInfo.java index 341f77f3c2d8..2d1f4df359a9 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHRegionInfo.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHRegionInfo.java @@ -17,41 +17,34 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; +import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; import java.util.function.Consumer; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; /** * Test ImmutableHRegionInfo */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestImmutableHRegionInfo { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestImmutableHRegionInfo.class); - - @Rule - public TestName name = new TestName(); - private final List> TEST_FUNCTIONS = Arrays.asList(hri -> hri.setOffline(true), hri -> hri.setSplit(true)); @Test - public void testImmutable() { - HRegionInfo hri = new HRegionInfo(TableName.valueOf(name.getMethodName())); + public void testImmutable(TestInfo testInfo) { + String methodName = testInfo.getTestMethod().map(Method::getName).orElse("default"); + HRegionInfo hri = new HRegionInfo(TableName.valueOf(methodName)); ImmutableHRegionInfo immutableHri = new ImmutableHRegionInfo(hri); TEST_FUNCTIONS.forEach(f -> { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHTableDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHTableDescriptor.java index be0649f3751d..0d55a1a4135a 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHTableDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableHTableDescriptor.java @@ -17,13 +17,12 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.function.Consumer; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; @@ -31,21 +30,15 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestImmutableHTableDescriptor { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestImmutableHTableDescriptor.class); - - @Rule - public TestName name = new TestName(); private static final List> TEST_FUNCTION = Arrays.asList( htd -> htd.setValue("a", "a"), htd -> htd.setValue(Bytes.toBytes("a"), Bytes.toBytes("a")), htd -> htd.setValue(new Bytes(Bytes.toBytes("a")), new Bytes(Bytes.toBytes("a"))), @@ -66,9 +59,16 @@ public class TestImmutableHTableDescriptor { } }); + private String methodName; + + @BeforeEach + public void setUp(TestInfo testInfo) { + methodName = testInfo.getTestMethod().get().getName(); + } + @Test public void testImmutable() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(methodName)); ImmutableHTableDescriptor immutableHtd = new ImmutableHTableDescriptor(htd); TEST_FUNCTION.forEach(f -> { try { @@ -81,7 +81,7 @@ public void testImmutable() { @Test public void testImmutableHColumnDescriptor() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(methodName)); htd.addFamily(new HColumnDescriptor(Bytes.toBytes("family"))); ImmutableHTableDescriptor immutableHtd = new ImmutableHTableDescriptor(htd); for (HColumnDescriptor hcd : immutableHtd.getColumnFamilies()) { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableScan.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableScan.java index aa05f5a3c7d8..9bb27b6e702e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableScan.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestImmutableScan.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -26,7 +26,6 @@ import java.util.Collections; import java.util.Map; import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.security.access.Permission; @@ -34,9 +33,8 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,13 +42,10 @@ /** * Small tests for ImmutableScan */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestImmutableScan { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestImmutableScan.class); - private static final Logger LOG = LoggerFactory.getLogger(TestImmutableScan.class); @Test diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestIncrement.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestIncrement.java index 309c1007ae09..b42f1b235f68 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestIncrement.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestIncrement.java @@ -17,25 +17,20 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Map; import java.util.NavigableMap; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestIncrement { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIncrement.class); - @Test public void testIncrementInstance() { final long expected = 13; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestInterfaceAlign.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestInterfaceAlign.java index ebdb7adc5887..cb48ee627429 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestInterfaceAlign.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestInterfaceAlign.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.Closeable; import java.lang.annotation.Annotation; @@ -27,20 +27,15 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.hbase.Abortable; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestInterfaceAlign { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestInterfaceAlign.class); - /** * Test methods name match up */ @@ -62,14 +57,14 @@ public void testAdminWithAsyncAdmin() { if (method.endsWith("Async")) { contains = asyncAdminMethodNames.contains(method.replace("Async", "")); } - assertTrue("Admin method " + method + " should in AsyncAdmin too", contains); + assertTrue(contains, "Admin method " + method + " should in AsyncAdmin too"); }); asyncAdminMethodNames.forEach(method -> { boolean contains = adminMethodNames.contains(method); if (!contains) { contains = adminMethodNames.contains(method + "Async"); } - assertTrue("AsyncAdmin method " + method + " should in Admin too", contains); + assertTrue(contains, "AsyncAdmin method " + method + " should in Admin too"); }); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java index 6da0ee5bfa98..b3d19794668f 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.codahale.metrics.Counter; import com.codahale.metrics.RatioGauge; @@ -28,14 +30,14 @@ import com.codahale.metrics.Timer; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.CallTimeoutException; import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException; @@ -44,16 +46,11 @@ import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; -import org.mockito.Mockito; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.params.provider.Arguments; import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; @@ -68,12 +65,11 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; -@RunWith(Parameterized.class) -@Category({ ClientTests.class, MetricsTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) +@HBaseParameterizedTestTemplate public class TestMetricsConnection { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsConnection.class); private static final Configuration conf = new Configuration(); private static MetricsConnection METRICS; @@ -82,27 +78,29 @@ public class TestMetricsConnection { private static final String MOCK_CONN_STR = "mocked-connection"; - @Parameter() public boolean tableMetricsEnabled; - @Parameters - public static List params() { - return Arrays.asList(false, true); + public TestMetricsConnection(boolean tableMetricsEnabled) { + this.tableMetricsEnabled = tableMetricsEnabled; } - @Before + public static Stream parameters() { + return Stream.of(Arguments.of(true), Arguments.of(false)); + } + + @BeforeEach public void before() { conf.setBoolean(MetricsConnection.CLIENT_SIDE_TABLE_METRICS_ENABLED_KEY, tableMetricsEnabled); METRICS = MetricsConnection.getMetricsConnection(conf, MOCK_CONN_STR, () -> BATCH_POOL, () -> null); } - @After + @AfterEach public void after() { MetricsConnection.deleteMetricsConnection(MOCK_CONN_STR); } - @Test + @TestTemplate public void testMetricsConnectionScopeAsyncClient() throws IOException { Configuration conf = new Configuration(); String clusterId = "foo"; @@ -111,18 +109,18 @@ public void testMetricsConnectionScopeAsyncClient() throws IOException { AsyncConnectionImpl impl = new AsyncConnectionImpl(conf, null, "foo", User.getCurrent()); Optional metrics = impl.getConnectionMetrics(); - assertTrue("Metrics should be present", metrics.isPresent()); + assertTrue(metrics.isPresent(), "Metrics should be present"); assertEquals(clusterId + "@" + Integer.toHexString(impl.hashCode()), metrics.get().getMetricScope()); conf.set(MetricsConnection.METRICS_SCOPE_KEY, scope); impl = new AsyncConnectionImpl(conf, null, "foo", User.getCurrent()); metrics = impl.getConnectionMetrics(); - assertTrue("Metrics should be present", metrics.isPresent()); + assertTrue(metrics.isPresent(), "Metrics should be present"); assertEquals(scope, metrics.get().getMetricScope()); } - @Test + @TestTemplate public void testMetricsWithMultiConnections() throws IOException { Configuration conf = new Configuration(); conf.setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true); @@ -142,11 +140,11 @@ public void testMetricsWithMultiConnections() throws IOException { /* verify metrics presence */ impl = connList.get(0); Optional metrics = impl.getConnectionMetrics(); - assertTrue("Metrics should be present", metrics.isPresent()); + assertTrue(metrics.isPresent(), "Metrics should be present"); /* verify connection count in a shared metrics */ long count = metrics.get().getConnectionCount(); - assertEquals("Failed to verify connection count." + count, count, num); + assertEquals(count, num, "Failed to verify connection count." + count); /* close some connections */ for (int i = 0; i < num - 1; i++) { @@ -156,42 +154,41 @@ public void testMetricsWithMultiConnections() throws IOException { /* verify metrics presence again */ impl = connList.get(num - 1); metrics = impl.getConnectionMetrics(); - assertTrue("Metrics should be present after some of connections are closed.", - metrics.isPresent()); + assertTrue(metrics.isPresent(), + "Metrics should be present after some of connections are closed."); /* verify count of remaining connections */ count = metrics.get().getConnectionCount(); - assertEquals("Connection count suppose to be 1 but got: " + count, count, 1); + assertEquals(count, 1, "Connection count suppose to be 1 but got: " + count); /* shutdown */ impl.close(); } - @Test + @TestTemplate public void testMetricsConnectionScopeBlockingClient() throws IOException { Configuration conf = new Configuration(); String clusterId = "foo"; String scope = "testScope"; conf.setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true); - ConnectionRegistry mockRegistry = Mockito.mock(ConnectionRegistry.class); - Mockito.when(mockRegistry.getClusterId()) - .thenReturn(CompletableFuture.completedFuture(clusterId)); + ConnectionRegistry mockRegistry = mock(ConnectionRegistry.class); + when(mockRegistry.getClusterId()).thenReturn(CompletableFuture.completedFuture(clusterId)); ConnectionImplementation impl = new ConnectionImplementation(conf, null, User.getCurrent(), mockRegistry); MetricsConnection metrics = impl.getConnectionMetrics(); - assertNotNull("Metrics should be present", metrics); + assertNotNull(metrics, "Metrics should be present"); assertEquals(clusterId + "@" + Integer.toHexString(impl.hashCode()), metrics.getMetricScope()); conf.set(MetricsConnection.METRICS_SCOPE_KEY, scope); impl = new ConnectionImplementation(conf, null, User.getCurrent(), mockRegistry); metrics = impl.getConnectionMetrics(); - assertNotNull("Metrics should be present", metrics); + assertNotNull(metrics, "Metrics should be present"); assertEquals(scope, metrics.getMetricScope()); } - @Test + @TestTemplate public void testStaticMetrics() throws IOException { final byte[] foo = Bytes.toBytes("foo"); String table = "TableX"; @@ -257,19 +254,19 @@ public void testStaticMetrics() throws IOException { metricKey = "rpcRemoteExceptions_IOException"; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; - assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop); + assertEquals(metricVal, loop, "metric: " + metricKey + " val: " + metricVal); // local exception metricKey = "rpcLocalExceptions_CallTimeoutException"; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; - assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop * 2); + assertEquals(metricVal, loop * 2, "metric: " + metricKey + " val: " + metricVal); // total exception metricKey = "rpcTotalExceptions"; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; - assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop * 3); + assertEquals(metricVal, loop * 3, "metric: " + metricKey + " val: " + metricVal); testRpcCallTableMetrics(table, loop); @@ -277,9 +274,9 @@ public void testStaticMetrics() throws IOException { METRICS.getGetTracker(), METRICS.getScanTracker(), METRICS.getMultiTracker(), METRICS.getAppendTracker(), METRICS.getDeleteTracker(), METRICS.getIncrementTracker(), METRICS.getPutTracker() }) { - assertEquals("Failed to invoke callTimer on " + t, loop, t.callTimer.getCount()); - assertEquals("Failed to invoke reqHist on " + t, loop, t.reqHist.getCount()); - assertEquals("Failed to invoke respHist on " + t, loop, t.respHist.getCount()); + assertEquals(loop, t.callTimer.getCount(), "Failed to invoke callTimer on " + t); + assertEquals(loop, t.reqHist.getCount(), "Failed to invoke reqHist on " + t); + assertEquals(loop, t.respHist.getCount(), "Failed to invoke respHist on " + t); } RatioGauge executorMetrics = (RatioGauge) METRICS.getMetricRegistry().getMetrics().get(METRICS.getExecutorPoolName()); @@ -303,10 +300,10 @@ private void testRpcCallTableMetrics(String table, int expectedVal) { long numOps = timer.getCount(); double p95 = timer.getSnapshot().get95thPercentile(); double p99 = timer.getSnapshot().get99thPercentile(); - assertEquals("metric: " + metricKey + numOpsSuffix + " val: " + numOps, expectedVal, - numOps); - assertTrue("metric: " + metricKey + p95Suffix + " val: " + p95, p95 >= 0); - assertTrue("metric: " + metricKey + p99Suffix + " val: " + p99, p99 >= 0); + assertEquals(expectedVal, numOps, + "metric: " + metricKey + numOpsSuffix + " val: " + numOps); + assertTrue(p95 >= 0, "metric: " + metricKey + p95Suffix + " val: " + p95); + assertTrue(p99 >= 0, "metric: " + metricKey + p99Suffix + " val: " + p99); } else { assertNull(timer); } @@ -322,10 +319,10 @@ private void testRpcCallTableMetrics(String table, int expectedVal) { long numOps = timer.getCount(); double p95 = timer.getSnapshot().get95thPercentile(); double p99 = timer.getSnapshot().get99thPercentile(); - assertEquals("metric: " + metricKey + numOpsSuffix + " val: " + numOps, expectedVal, - numOps); - assertTrue("metric: " + metricKey + p95Suffix + " val: " + p95, p95 >= 0); - assertTrue("metric: " + metricKey + p99Suffix + " val: " + p99, p99 >= 0); + assertEquals(expectedVal, numOps, + "metric: " + metricKey + numOpsSuffix + " val: " + numOps); + assertTrue(p95 >= 0, "metric: " + metricKey + p95Suffix + " val: " + p95); + assertTrue(p99 >= 0, "metric: " + metricKey + p99Suffix + " val: " + p99); } else { assertNull(timer); } @@ -344,7 +341,7 @@ private void testRpcCallMetrics(String table, int expectedVal) { // rpc call count metricKey = rpcCountPrefix + method; metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); - assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, expectedVal); + assertEquals(metricVal, expectedVal, "metric: " + metricKey + " val: " + metricVal); // rpc failure call metricKey = tableMetricsEnabled @@ -354,10 +351,10 @@ private void testRpcCallMetrics(String table, int expectedVal) { metricVal = (counter != null) ? counter.getCount() : 0; if (method.equals("Get")) { // no failure - assertEquals("metric: " + metricKey + " val: " + metricVal, 0, metricVal); + assertEquals(0, metricVal, "metric: " + metricKey + " val: " + metricVal); } else { // has failure - assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, expectedVal); + assertEquals(metricVal, expectedVal, "metric: " + metricKey + " val: " + metricVal); } } @@ -366,7 +363,7 @@ private void testRpcCallMetrics(String table, int expectedVal) { // rpc call count metricKey = rpcCountPrefix + method + "(" + mutationType + ")"; metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); - assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, expectedVal); + assertEquals(metricVal, expectedVal, "metric: " + metricKey + " val: " + metricVal); // rpc failure call metricKey = tableMetricsEnabled @@ -376,10 +373,10 @@ private void testRpcCallMetrics(String table, int expectedVal) { metricVal = (counter != null) ? counter.getCount() : 0; if (mutationType.equals("Put")) { // has failure - assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, expectedVal); + assertEquals(metricVal, expectedVal, "metric: " + metricKey + " val: " + metricVal); } else { // no failure - assertEquals("metric: " + metricKey + " val: " + metricVal, 0, metricVal); + assertEquals(0, metricVal, "metric: " + metricKey + " val: " + metricVal); } } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMutation.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMutation.java index 718ca05b92f3..acc85fc4e1f7 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMutation.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMutation.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.List; @@ -26,23 +26,18 @@ import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ SmallTests.class, ClientTests.class }) +@Tag(SmallTests.TAG) +@Tag(ClientTests.TAG) public class TestMutation { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMutation.class); - @Test public void testAppendCopyConstructor() throws IOException { Append origin = new Append(Bytes.toBytes("ROW-01")); @@ -122,11 +117,11 @@ public void testPutCopyConstructor() throws IOException { } private void assertEquals(Mutation origin, Mutation clone) { - Assert.assertEquals(origin.getFamilyCellMap().size(), clone.getFamilyCellMap().size()); + Assertions.assertEquals(origin.getFamilyCellMap().size(), clone.getFamilyCellMap().size()); for (byte[] family : origin.getFamilyCellMap().keySet()) { List originCells = origin.getCellList(family); List cloneCells = clone.getCellList(family); - Assert.assertEquals(originCells.size(), cloneCells.size()); + Assertions.assertEquals(originCells.size(), cloneCells.size()); for (int i = 0; i != cloneCells.size(); ++i) { Cell originCell = originCells.get(i); Cell cloneCell = cloneCells.get(i); @@ -134,14 +129,14 @@ private void assertEquals(Mutation origin, Mutation clone) { assertTrue(CellUtil.matchingValue(originCell, cloneCell)); } } - Assert.assertEquals(origin.getAttributesMap().size(), clone.getAttributesMap().size()); + Assertions.assertEquals(origin.getAttributesMap().size(), clone.getAttributesMap().size()); for (String name : origin.getAttributesMap().keySet()) { byte[] originValue = origin.getAttributesMap().get(name); byte[] cloneValue = clone.getAttributesMap().get(name); assertTrue(Bytes.equals(originValue, cloneValue)); } - Assert.assertEquals(origin.getTimestamp(), clone.getTimestamp()); - Assert.assertEquals(origin.getPriority(), clone.getPriority()); + Assertions.assertEquals(origin.getTimestamp(), clone.getTimestamp()); + Assertions.assertEquals(origin.getPriority(), clone.getPriority()); if (origin instanceof Append) { assertEquals(((Append) origin).getTimeRange(), ((Append) clone).getTimeRange()); } @@ -151,8 +146,8 @@ private void assertEquals(Mutation origin, Mutation clone) { } private static void assertEquals(TimeRange origin, TimeRange clone) { - Assert.assertEquals(origin.getMin(), clone.getMin()); - Assert.assertEquals(origin.getMax(), clone.getMax()); + Assertions.assertEquals(origin.getMin(), clone.getMin()); + Assertions.assertEquals(origin.getMax(), clone.getMax()); } // HBASE-14881 diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java index 4446ab0f7261..b2a8cf94d1c9 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java @@ -17,28 +17,26 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Collections; import java.util.Map; import java.util.Set; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestOnlineLogRecord { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestOnlineLogRecord.class); - @Test public void itSerializesScan() { Scan scan = new Scan(); @@ -63,7 +61,7 @@ public void itSerializesScan() { null, 6, 7, 0, scan, Collections.emptyMap(), Collections.emptyMap()); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); - Assert.assertEquals(expectedOutput, actualOutput); + assertEquals(expectedOutput, actualOutput); } @Test @@ -77,7 +75,7 @@ ImmutableSet. builder().add("requestAttributes").add("\"r\": \"1\"") null, 6, 7, 0, null, requestAttributes, Collections.emptyMap()); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); - expectedOutputs.forEach(expected -> Assert.assertTrue(actualOutput.contains(expected))); + expectedOutputs.forEach(expected -> assertTrue(actualOutput.contains(expected))); } @Test @@ -86,7 +84,7 @@ public void itOmitsEmptyRequestAttributes() { null, 6, 7, 0, null, Collections.emptyMap(), Collections.emptyMap()); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); - Assert.assertFalse(actualOutput.contains("requestAttributes")); + assertFalse(actualOutput.contains("requestAttributes")); } @Test @@ -100,7 +98,7 @@ ImmutableSet. builder().add("connectionAttributes").add("\"c\": \"1\"") null, 6, 7, 0, null, Collections.emptyMap(), connectionAttributes); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); - expectedOutputs.forEach(expected -> Assert.assertTrue(actualOutput.contains(expected))); + expectedOutputs.forEach(expected -> assertTrue(actualOutput.contains(expected))); } @Test @@ -109,6 +107,6 @@ public void itOmitsEmptyConnectionAttributes() { null, 6, 7, 0, null, Collections.emptyMap(), Collections.emptyMap()); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); - Assert.assertFalse(actualOutput.contains("connectionAttributes")); + assertFalse(actualOutput.contains("connectionAttributes")); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java index 644062ca9917..170a569d5baa 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.IOException; import java.lang.reflect.Type; @@ -29,7 +30,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.filter.BinaryComparator; @@ -62,10 +62,8 @@ import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.GsonUtil; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.reflect.TypeToken; import org.apache.hbase.thirdparty.com.google.gson.Gson; @@ -77,11 +75,9 @@ * Run tests that use the functionality of the Operation superclass for Puts, Gets, Deletes, Scans, * and MultiPuts. */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestOperation { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestOperation.class); private static byte[] ROW = Bytes.toBytes("testRow"); private static byte[] FAMILY = Bytes.toBytes("testFamily"); @@ -269,8 +265,8 @@ public class TestOperation { }; static { - assertEquals("The sizes of static arrays do not match: " + "[FILTERS: %d <=> FILTERS_INFO: %d]", - FILTERS.length, FILTERS_INFO.length); + assertEquals(FILTERS.length, FILTERS_INFO.length, + "The sizes of static arrays do not match: " + "[FILTERS: %d <=> FILTERS_INFO: %d]"); } /** @@ -289,14 +285,14 @@ public void testOperationJSON() throws IOException { }.getType(); Map parsedJSON = GSON.fromJson(json, typeOfHashMap); // check for the row - assertEquals("startRow incorrect in Scan.toJSON()", Bytes.toStringBinary(ROW), - parsedJSON.get("startRow")); + assertEquals(Bytes.toStringBinary(ROW), parsedJSON.get("startRow"), + "startRow incorrect in Scan.toJSON()"); // check for the family and the qualifier. List familyInfo = (List) ((Map) parsedJSON.get("families")).get(Bytes.toStringBinary(FAMILY)); - assertNotNull("Family absent in Scan.toJSON()", familyInfo); - assertEquals("Qualifier absent in Scan.toJSON()", 1, familyInfo.size()); - assertEquals("Qualifier incorrect in Scan.toJSON()", Bytes.toStringBinary(QUALIFIER), - familyInfo.get(0)); + assertNotNull(familyInfo, "Family absent in Scan.toJSON()"); + assertEquals(1, familyInfo.size(), "Qualifier absent in Scan.toJSON()"); + assertEquals(Bytes.toStringBinary(QUALIFIER), familyInfo.get(0), + "Qualifier incorrect in Scan.toJSON()"); // produce a Get Operation Get get = new Get(ROW); @@ -305,13 +301,13 @@ public void testOperationJSON() throws IOException { json = get.toJSON(); parsedJSON = GSON.fromJson(json, typeOfHashMap); // check for the row - assertEquals("row incorrect in Get.toJSON()", Bytes.toStringBinary(ROW), parsedJSON.get("row")); + assertEquals(Bytes.toStringBinary(ROW), parsedJSON.get("row"), "row incorrect in Get.toJSON()"); // check for the family and the qualifier. familyInfo = (List) ((Map) parsedJSON.get("families")).get(Bytes.toStringBinary(FAMILY)); - assertNotNull("Family absent in Get.toJSON()", familyInfo); - assertEquals("Qualifier absent in Get.toJSON()", 1, familyInfo.size()); - assertEquals("Qualifier incorrect in Get.toJSON()", Bytes.toStringBinary(QUALIFIER), - familyInfo.get(0)); + assertNotNull(familyInfo, "Family absent in Get.toJSON()"); + assertEquals(1, familyInfo.size(), "Qualifier absent in Get.toJSON()"); + assertEquals(Bytes.toStringBinary(QUALIFIER), familyInfo.get(0), + "Qualifier incorrect in Get.toJSON()"); // produce a Put operation Put put = new Put(ROW); @@ -320,16 +316,16 @@ public void testOperationJSON() throws IOException { json = put.toJSON(); parsedJSON = GSON.fromJson(json, typeOfHashMap); // check for the row - assertEquals("row absent in Put.toJSON()", Bytes.toStringBinary(ROW), parsedJSON.get("row")); + assertEquals(Bytes.toStringBinary(ROW), parsedJSON.get("row"), "row absent in Put.toJSON()"); // check for the family and the qualifier. familyInfo = (List) ((Map) parsedJSON.get("families")).get(Bytes.toStringBinary(FAMILY)); - assertNotNull("Family absent in Put.toJSON()", familyInfo); - assertEquals("KeyValue absent in Put.toJSON()", 1, familyInfo.size()); + assertNotNull(familyInfo, "Family absent in Put.toJSON()"); + assertEquals(1, familyInfo.size(), "KeyValue absent in Put.toJSON()"); Map kvMap = (Map) familyInfo.get(0); - assertEquals("Qualifier incorrect in Put.toJSON()", Bytes.toStringBinary(QUALIFIER), - kvMap.get("qualifier")); - assertEquals("Value length incorrect in Put.toJSON()", VALUE.length, - ((Number) kvMap.get("vlen")).intValue()); + assertEquals(Bytes.toStringBinary(QUALIFIER), kvMap.get("qualifier"), + "Qualifier incorrect in Put.toJSON()"); + assertEquals(VALUE.length, ((Number) kvMap.get("vlen")).intValue(), + "Value length incorrect in Put.toJSON()"); // produce a Delete operation Delete delete = new Delete(ROW); @@ -338,14 +334,14 @@ public void testOperationJSON() throws IOException { json = delete.toJSON(); parsedJSON = GSON.fromJson(json, typeOfHashMap); // check for the row - assertEquals("row absent in Delete.toJSON()", Bytes.toStringBinary(ROW), parsedJSON.get("row")); + assertEquals(Bytes.toStringBinary(ROW), parsedJSON.get("row"), "row absent in Delete.toJSON()"); // check for the family and the qualifier. familyInfo = (List) ((Map) parsedJSON.get("families")).get(Bytes.toStringBinary(FAMILY)); - assertNotNull("Family absent in Delete.toJSON()", familyInfo); - assertEquals("KeyValue absent in Delete.toJSON()", 1, familyInfo.size()); + assertNotNull(familyInfo, "Family absent in Delete.toJSON()"); + assertEquals(1, familyInfo.size(), "KeyValue absent in Delete.toJSON()"); kvMap = (Map) familyInfo.get(0); - assertEquals("Qualifier incorrect in Delete.toJSON()", Bytes.toStringBinary(QUALIFIER), - kvMap.get("qualifier")); + assertEquals(Bytes.toStringBinary(QUALIFIER), kvMap.get("qualifier"), + "Qualifier incorrect in Delete.toJSON()"); } /** @@ -389,57 +385,57 @@ public void testScanOperationToJSON() throws IOException { .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create(); Map parsedJSON = gson.fromJson(json, typeOfHashMap); // check for the row - assertEquals("startRow incorrect in Scan.toJSON()", Bytes.toStringBinary(ROW), - parsedJSON.get("startRow")); + assertEquals(Bytes.toStringBinary(ROW), parsedJSON.get("startRow"), + "startRow incorrect in Scan.toJSON()"); // check for the family and the qualifier. List familyInfo = (List) ((Map) parsedJSON.get("families")).get(Bytes.toStringBinary(FAMILY)); - assertNotNull("Family absent in Scan.toJSON()", familyInfo); - assertEquals("Qualifier absent in Scan.toJSON()", 1, familyInfo.size()); - assertEquals("Qualifier incorrect in Scan.toJSON()", Bytes.toStringBinary(QUALIFIER), - familyInfo.get(0)); - assertEquals("stopRow incorrect in Scan.toJSON()", Bytes.toStringBinary(ROW), - parsedJSON.get("stopRow")); - assertEquals("includeStartRow incorrect in Scan.toJSON()", true, - parsedJSON.get("includeStartRow")); - assertEquals("includeStopRow incorrect in Scan.toJSON()", true, - parsedJSON.get("includeStopRow")); - assertEquals("maxVersions incorrect in Scan.toJSON()", 5L, parsedJSON.get("maxVersions")); - assertEquals("batch incorrect in Scan.toJSON()", 10L, parsedJSON.get("batch")); - assertEquals("allowPartialResults incorrect in Scan.toJSON()", true, - parsedJSON.get("allowPartialResults")); - assertEquals("storeLimit incorrect in Scan.toJSON()", 3L, parsedJSON.get("storeLimit")); - assertEquals("storeOffset incorrect in Scan.toJSON()", 8L, parsedJSON.get("storeOffset")); - assertEquals("caching incorrect in Scan.toJSON()", 20L, parsedJSON.get("caching")); - assertEquals("maxResultSize incorrect in Scan.toJSON()", "50", parsedJSON.get("maxResultSize")); - assertEquals("cacheBlocks incorrect in Scan.toJSON()", true, parsedJSON.get("cacheBlocks")); - assertEquals("reversed incorrect in Scan.toJSON()", true, parsedJSON.get("reversed")); + assertNotNull(familyInfo, "Family absent in Scan.toJSON()"); + assertEquals(1, familyInfo.size(), "Qualifier absent in Scan.toJSON()"); + assertEquals(Bytes.toStringBinary(QUALIFIER), familyInfo.get(0), + "Qualifier incorrect in Scan.toJSON()"); + assertEquals(Bytes.toStringBinary(ROW), parsedJSON.get("stopRow"), + "stopRow incorrect in Scan.toJSON()"); + assertEquals(true, parsedJSON.get("includeStartRow"), + "includeStartRow incorrect in Scan.toJSON()"); + assertEquals(true, parsedJSON.get("includeStopRow"), + "includeStopRow incorrect in Scan.toJSON()"); + assertEquals(5L, parsedJSON.get("maxVersions"), "maxVersions incorrect in Scan.toJSON()"); + assertEquals(10L, parsedJSON.get("batch"), "batch incorrect in Scan.toJSON()"); + assertEquals(true, parsedJSON.get("allowPartialResults"), + "allowPartialResults incorrect in Scan.toJSON()"); + assertEquals(3L, parsedJSON.get("storeLimit"), "storeLimit incorrect in Scan.toJSON()"); + assertEquals(8L, parsedJSON.get("storeOffset"), "storeOffset incorrect in Scan.toJSON()"); + assertEquals(20L, parsedJSON.get("caching"), "caching incorrect in Scan.toJSON()"); + assertEquals("50", parsedJSON.get("maxResultSize"), "maxResultSize incorrect in Scan.toJSON()"); + assertEquals(true, parsedJSON.get("cacheBlocks"), "cacheBlocks incorrect in Scan.toJSON()"); + assertEquals(true, parsedJSON.get("reversed"), "reversed incorrect in Scan.toJSON()"); List trList = (List) parsedJSON.get("timeRange"); - assertEquals("timeRange incorrect in Scan.toJSON()", 2, trList.size()); - assertEquals("timeRange incorrect in Scan.toJSON()", "1000", trList.get(0)); - assertEquals("timeRange incorrect in Scan.toJSON()", "2000", trList.get(1)); - - assertEquals("asyncPrefetch incorrect in Scan.toJSON()", true, parsedJSON.get("asyncPrefetch")); - assertEquals("mvccReadPoint incorrect in Scan.toJSON()", "123", - parsedJSON.get("mvccReadPoint")); - assertEquals("limit incorrect in Scan.toJSON()", 5L, parsedJSON.get("limit")); - assertEquals("readType incorrect in Scan.toJSON()", "PREAD", parsedJSON.get("readType")); - assertEquals("needCursorResult incorrect in Scan.toJSON()", true, - parsedJSON.get("needCursorResult")); + assertEquals(2, trList.size(), "timeRange incorrect in Scan.toJSON()"); + assertEquals("1000", trList.get(0), "timeRange incorrect in Scan.toJSON()"); + assertEquals("2000", trList.get(1), "timeRange incorrect in Scan.toJSON()"); + + assertEquals(true, parsedJSON.get("asyncPrefetch"), "asyncPrefetch incorrect in Scan.toJSON()"); + assertEquals("123", parsedJSON.get("mvccReadPoint"), + "mvccReadPoint incorrect in Scan.toJSON()"); + assertEquals(5L, parsedJSON.get("limit"), "limit incorrect in Scan.toJSON()"); + assertEquals("PREAD", parsedJSON.get("readType"), "readType incorrect in Scan.toJSON()"); + assertEquals(true, parsedJSON.get("needCursorResult"), + "needCursorResult incorrect in Scan.toJSON()"); Map colFamTimeRange = (Map) parsedJSON.get("colFamTimeRangeMap"); - assertEquals("colFamTimeRangeMap incorrect in Scan.toJSON()", 1L, colFamTimeRange.size()); + assertEquals(1L, colFamTimeRange.size(), "colFamTimeRangeMap incorrect in Scan.toJSON()"); List testFamily = (List) colFamTimeRange.get("testFamily"); - assertEquals("colFamTimeRangeMap incorrect in Scan.toJSON()", 2L, testFamily.size()); - assertEquals("colFamTimeRangeMap incorrect in Scan.toJSON()", "2000", testFamily.get(0)); - assertEquals("colFamTimeRangeMap incorrect in Scan.toJSON()", "3000", testFamily.get(1)); + assertEquals(2L, testFamily.size(), "colFamTimeRangeMap incorrect in Scan.toJSON()"); + assertEquals("2000", testFamily.get(0), "colFamTimeRangeMap incorrect in Scan.toJSON()"); + assertEquals("3000", testFamily.get(1), "colFamTimeRangeMap incorrect in Scan.toJSON()"); - assertEquals("targetReplicaId incorrect in Scan.toJSON()", 1L, - parsedJSON.get("targetReplicaId")); - assertEquals("consistency incorrect in Scan.toJSON()", "STRONG", parsedJSON.get("consistency")); - assertEquals("loadColumnFamiliesOnDemand incorrect in Scan.toJSON()", true, - parsedJSON.get("loadColumnFamiliesOnDemand")); + assertEquals(1L, parsedJSON.get("targetReplicaId"), + "targetReplicaId incorrect in Scan.toJSON()"); + assertEquals("STRONG", parsedJSON.get("consistency"), "consistency incorrect in Scan.toJSON()"); + assertEquals(true, parsedJSON.get("loadColumnFamiliesOnDemand"), + "loadColumnFamiliesOnDemand incorrect in Scan.toJSON()"); - assertEquals("priority incorrect in Scan.toJSON()", 10L, parsedJSON.get("priority")); + assertEquals(10L, parsedJSON.get("priority"), "priority incorrect in Scan.toJSON()"); } @@ -447,45 +443,45 @@ public void testScanOperationToJSON() throws IOException { public void testPutCreationWithByteBuffer() { Put p = new Put(ROW); List c = p.get(FAMILY, QUALIFIER); - Assert.assertEquals(0, c.size()); - Assert.assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimestamp()); + assertEquals(0, c.size()); + assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimestamp()); p.addColumn(FAMILY, ByteBuffer.wrap(QUALIFIER), 1984L, ByteBuffer.wrap(VALUE)); c = p.get(FAMILY, QUALIFIER); - Assert.assertEquals(1, c.size()); - Assert.assertEquals(1984L, c.get(0).getTimestamp()); - Assert.assertArrayEquals(VALUE, CellUtil.cloneValue(c.get(0))); - Assert.assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimestamp()); - Assert.assertEquals(0, CellComparatorImpl.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0)))); + assertEquals(1, c.size()); + assertEquals(1984L, c.get(0).getTimestamp()); + assertArrayEquals(VALUE, CellUtil.cloneValue(c.get(0))); + assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimestamp()); + assertEquals(0, CellComparatorImpl.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0)))); p = new Put(ROW); p.addColumn(FAMILY, ByteBuffer.wrap(QUALIFIER), 2013L, null); c = p.get(FAMILY, QUALIFIER); - Assert.assertEquals(1, c.size()); - Assert.assertEquals(2013L, c.get(0).getTimestamp()); - Assert.assertArrayEquals(new byte[] {}, CellUtil.cloneValue(c.get(0))); - Assert.assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimestamp()); - Assert.assertEquals(0, CellComparatorImpl.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0)))); + assertEquals(1, c.size()); + assertEquals(2013L, c.get(0).getTimestamp()); + assertArrayEquals(new byte[] {}, CellUtil.cloneValue(c.get(0))); + assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimestamp()); + assertEquals(0, CellComparatorImpl.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0)))); p = new Put(ByteBuffer.wrap(ROW)); p.addColumn(FAMILY, ByteBuffer.wrap(QUALIFIER), 2001L, null); c = p.get(FAMILY, QUALIFIER); - Assert.assertEquals(1, c.size()); - Assert.assertEquals(2001L, c.get(0).getTimestamp()); - Assert.assertArrayEquals(new byte[] {}, CellUtil.cloneValue(c.get(0))); - Assert.assertArrayEquals(ROW, CellUtil.cloneRow(c.get(0))); - Assert.assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimestamp()); - Assert.assertEquals(0, CellComparatorImpl.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0)))); + assertEquals(1, c.size()); + assertEquals(2001L, c.get(0).getTimestamp()); + assertArrayEquals(new byte[] {}, CellUtil.cloneValue(c.get(0))); + assertArrayEquals(ROW, CellUtil.cloneRow(c.get(0))); + assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimestamp()); + assertEquals(0, CellComparatorImpl.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0)))); p = new Put(ByteBuffer.wrap(ROW), 1970L); p.addColumn(FAMILY, ByteBuffer.wrap(QUALIFIER), 2001L, null); c = p.get(FAMILY, QUALIFIER); - Assert.assertEquals(1, c.size()); - Assert.assertEquals(2001L, c.get(0).getTimestamp()); - Assert.assertArrayEquals(new byte[] {}, CellUtil.cloneValue(c.get(0))); - Assert.assertArrayEquals(ROW, CellUtil.cloneRow(c.get(0))); - Assert.assertEquals(1970L, p.getTimestamp()); - Assert.assertEquals(0, CellComparatorImpl.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0)))); + assertEquals(1, c.size()); + assertEquals(2001L, c.get(0).getTimestamp()); + assertArrayEquals(new byte[] {}, CellUtil.cloneValue(c.get(0))); + assertArrayEquals(ROW, CellUtil.cloneRow(c.get(0))); + assertEquals(1970L, p.getTimestamp()); + assertEquals(0, CellComparatorImpl.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0)))); } @Test @@ -538,42 +534,42 @@ public void testGetOperationToJSON() throws IOException { .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create(); Map parsedJSON = gson.fromJson(json, typeOfHashMap); // check for the row - assertEquals("row incorrect in Get.toJSON()", Bytes.toStringBinary(ROW), parsedJSON.get("row")); + assertEquals(Bytes.toStringBinary(ROW), parsedJSON.get("row"), "row incorrect in Get.toJSON()"); // check for the family and the qualifier. List familyInfo = (List) ((Map) parsedJSON.get("families")).get(Bytes.toStringBinary(FAMILY)); - assertNotNull("Family absent in Get.toJSON()", familyInfo); - assertEquals("Qualifier absent in Get.toJSON()", 1, familyInfo.size()); - assertEquals("Qualifier incorrect in Get.toJSON()", Bytes.toStringBinary(QUALIFIER), - familyInfo.get(0)); + assertNotNull(familyInfo, "Family absent in Get.toJSON()"); + assertEquals(1, familyInfo.size(), "Qualifier absent in Get.toJSON()"); + assertEquals(Bytes.toStringBinary(QUALIFIER), familyInfo.get(0), + "Qualifier incorrect in Get.toJSON()"); - assertEquals("maxVersions incorrect in Get.toJSON()", 5L, parsedJSON.get("maxVersions")); + assertEquals(5L, parsedJSON.get("maxVersions"), "maxVersions incorrect in Get.toJSON()"); - assertEquals("storeLimit incorrect in Get.toJSON()", 5L, parsedJSON.get("storeLimit")); - assertEquals("storeOffset incorrect in Get.toJSON()", 9L, parsedJSON.get("storeOffset")); + assertEquals(5L, parsedJSON.get("storeLimit"), "storeLimit incorrect in Get.toJSON()"); + assertEquals(9L, parsedJSON.get("storeOffset"), "storeOffset incorrect in Get.toJSON()"); - assertEquals("cacheBlocks incorrect in Get.toJSON()", true, parsedJSON.get("cacheBlocks")); + assertEquals(true, parsedJSON.get("cacheBlocks"), "cacheBlocks incorrect in Get.toJSON()"); List trList = (List) parsedJSON.get("timeRange"); - assertEquals("timeRange incorrect in Get.toJSON()", 2, trList.size()); - assertEquals("timeRange incorrect in Get.toJSON()", "1000", trList.get(0)); - assertEquals("timeRange incorrect in Get.toJSON()", "2000", trList.get(1)); + assertEquals(2, trList.size(), "timeRange incorrect in Get.toJSON()"); + assertEquals("1000", trList.get(0), "timeRange incorrect in Get.toJSON()"); + assertEquals("2000", trList.get(1), "timeRange incorrect in Get.toJSON()"); Map colFamTimeRange = (Map) parsedJSON.get("colFamTimeRangeMap"); - assertEquals("colFamTimeRangeMap incorrect in Get.toJSON()", 1L, colFamTimeRange.size()); + assertEquals(1L, colFamTimeRange.size(), "colFamTimeRangeMap incorrect in Get.toJSON()"); List testFamily = (List) colFamTimeRange.get("testFamily"); - assertEquals("colFamTimeRangeMap incorrect in Get.toJSON()", 2L, testFamily.size()); - assertEquals("colFamTimeRangeMap incorrect in Get.toJSON()", "2000", testFamily.get(0)); - assertEquals("colFamTimeRangeMap incorrect in Get.toJSON()", "3000", testFamily.get(1)); - - assertEquals("targetReplicaId incorrect in Get.toJSON()", 1L, - parsedJSON.get("targetReplicaId")); - assertEquals("consistency incorrect in Get.toJSON()", "STRONG", parsedJSON.get("consistency")); - assertEquals("loadColumnFamiliesOnDemand incorrect in Get.toJSON()", true, - parsedJSON.get("loadColumnFamiliesOnDemand")); - - assertEquals("priority incorrect in Get.toJSON()", 10L, parsedJSON.get("priority")); - assertEquals("checkExistenceOnly incorrect in Get.toJSON()", true, - parsedJSON.get("checkExistenceOnly")); + assertEquals(2L, testFamily.size(), "colFamTimeRangeMap incorrect in Get.toJSON()"); + assertEquals("2000", testFamily.get(0), "colFamTimeRangeMap incorrect in Get.toJSON()"); + assertEquals("3000", testFamily.get(1), "colFamTimeRangeMap incorrect in Get.toJSON()"); + + assertEquals(1L, parsedJSON.get("targetReplicaId"), + "targetReplicaId incorrect in Get.toJSON()"); + assertEquals("STRONG", parsedJSON.get("consistency"), "consistency incorrect in Get.toJSON()"); + assertEquals(true, parsedJSON.get("loadColumnFamiliesOnDemand"), + "loadColumnFamiliesOnDemand incorrect in Get.toJSON()"); + + assertEquals(10L, parsedJSON.get("priority"), "priority incorrect in Get.toJSON()"); + assertEquals(true, parsedJSON.get("checkExistenceOnly"), + "checkExistenceOnly incorrect in Get.toJSON()"); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperationInterceptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperationInterceptor.java new file mode 100644 index 000000000000..ddafd774f941 --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperationInterceptor.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) +public class TestOperationInterceptor { + + private static class MockOperationInterceptor extends OperationInterceptor { + final List events = new ArrayList<>(); + volatile Throwable lastException; + volatile Object lastResult; + + public MockOperationInterceptor() { + super(); + } + + @Override + public void beforeAttempt(RetryingCallable callable) { + events.add("beforeAttempt"); + } + + @Override + public void afterAttemptSuccess(RetryingCallable callable, Object result) { + events.add("afterAttemptSuccess"); + this.lastResult = result; + } + + @Override + public void afterAttemptFailure(RetryingCallable callable, Throwable cause) { + events.add("afterAttemptFailure"); + this.lastException = cause; + } + + @Override + public void afterOperation() { + events.add("afterOperation"); + } + } + + private static class TestOperationInterceptorFactory implements OperationInterceptorFactory { + final AtomicInteger createCount = new AtomicInteger(0); + + @Override + public OperationInterceptor createInterceptor() { + createCount.incrementAndGet(); + return new MockOperationInterceptor(); + } + } + + // Helper method to create RpcRetryingCallerImpl with test factory + private RpcRetryingCallerImpl createCaller(TestOperationInterceptorFactory factory) { + return new RpcRetryingCallerImpl<>(100, 500, 3, + RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, 0, 0, null, factory); + } + + // Helper method to create RpcRetryingCallerImpl with fast retry settings + private RpcRetryingCallerImpl + createFastRetryCaller(TestOperationInterceptorFactory factory) { + return new RpcRetryingCallerImpl<>(10, 50, 2, + RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, 0, 0, null, factory); + } + + // Base RetryingCallable implementation with common no-op methods + private static abstract class BaseRetryingCallable implements RetryingCallable { + @Override + public void prepare(boolean reload) { + } + + @Override + public void throwable(Throwable t, boolean retrying) { + } + + @Override + public String getExceptionMessageAdditionalDetail() { + return null; + } + + @Override + public long sleep(long pause, int tries) { + return tries == 0 ? 1 : 0; // Short sleep for tests + } + } + + // Simple success callable + private static class SuccessCallable extends BaseRetryingCallable { + private final String result; + + SuccessCallable(String result) { + this.result = result; + } + + @Override + public String call(int callTimeout) { + return result; + } + } + + // Simple failure callable + private static class FailureCallable extends BaseRetryingCallable { + private final Exception exception; + + FailureCallable(Exception exception) { + this.exception = exception; + } + + @Override + public String call(int callTimeout) throws Exception { + throw exception; + } + } + + // Callable that fails on first attempt, succeeds on second + private static class RetrySuccessCallable extends BaseRetryingCallable { + private final AtomicInteger callCount = new AtomicInteger(0); + private final String successResult; + + RetrySuccessCallable(String successResult) { + this.successResult = successResult; + } + + @Override + public String call(int callTimeout) throws Exception { + int attempt = callCount.incrementAndGet(); + if (attempt == 1) { + throw new IOException("first failure"); + } + return successResult + " " + attempt; + } + + int getCallCount() { + return callCount.get(); + } + } + + @Test + public void testSuccessfulSingleOperation() throws Exception { + TestOperationInterceptorFactory factory = new TestOperationInterceptorFactory(); + RpcRetryingCallerImpl caller = createCaller(factory); + + String result = caller.callWithoutRetries(new SuccessCallable("success"), 1000); + assertEquals("success", result); + assertEquals(1, factory.createCount.get()); + } + + @Test + public void testFailedSingleOperation() throws Exception { + TestOperationInterceptorFactory factory = new TestOperationInterceptorFactory(); + RpcRetryingCallerImpl caller = createCaller(factory); + + try { + caller.callWithoutRetries(new FailureCallable(new IOException("test failure")), 1000); + fail("Expected IOException"); + } catch (IOException e) { + assertEquals("test failure", e.getMessage()); + } + assertEquals(1, factory.createCount.get()); + } + + @Test + public void testRetryingOperation() throws Exception { + TestOperationInterceptorFactory factory = new TestOperationInterceptorFactory(); + RpcRetryingCallerImpl caller = createFastRetryCaller(factory); + + RetrySuccessCallable callable = new RetrySuccessCallable("success on attempt"); + String result = caller.callWithRetries(callable, 5000); + assertEquals("success on attempt 2", result); + assertEquals(1, factory.createCount.get()); + assertEquals(2, callable.getCallCount()); + } + + @Test + public void testNoOpFactory() { + OperationInterceptorFactory factory = OperationInterceptorFactory.NO_OP; + OperationInterceptor interceptor = factory.createInterceptor(); + assertNotNull(interceptor); + + // Verify it creates new instances each time (no longer singleton) + OperationInterceptor interceptor2 = factory.createInterceptor(); + assertNotNull(interceptor2); + // They should be different instances + assertNotSame(interceptor, interceptor2); + } + + @Test + public void testConfigurationBasedFactory() { + Configuration conf = new Configuration(); + conf.set(OperationInterceptorFactory.HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL, + TestOperationInterceptorFactory.class.getName()); + + // Test factory creation + String clazz = conf.get(OperationInterceptorFactory.HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL); + assertNotNull(clazz); + assertEquals(TestOperationInterceptorFactory.class.getName(), clazz); + } +} diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestProcedureFuture.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestProcedureFuture.java index d066a72c60ca..e1685026bd6f 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestProcedureFuture.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestProcedureFuture.java @@ -17,33 +17,28 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetProcedureResultRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetProcedureResultResponse; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureFuture { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureFuture.class); - private static class TestFuture extends HBaseAdmin.ProcedureFuture { private boolean postOperationResultCalled = false; private boolean waitOperationResultCalled = false; @@ -108,10 +103,10 @@ public void testWithProcId() throws Exception { TestFuture f = new TestFuture(admin, 100L); f.get(1, TimeUnit.MINUTES); - assertTrue("expected getProcedureResult() to be called", f.wasGetProcedureResultCalled()); - assertTrue("expected convertResult() to be called", f.wasConvertResultCalled()); - assertFalse("unexpected waitOperationResult() called", f.wasWaitOperationResultCalled()); - assertTrue("expected postOperationResult() to be called", f.wasPostOperationResultCalled()); + assertTrue(f.wasGetProcedureResultCalled(), "expected getProcedureResult() to be called"); + assertTrue(f.wasConvertResultCalled(), "expected convertResult() to be called"); + assertFalse(f.wasWaitOperationResultCalled(), "unexpected waitOperationResult() called"); + assertTrue(f.wasPostOperationResultCalled(), "expected postOperationResult() to be called"); } /** @@ -136,9 +131,9 @@ public void testWithProcIdAndSpinning() throws Exception { f.get(1, TimeUnit.MINUTES); assertEquals(10, spinCount.get()); - assertTrue("expected convertResult() to be called", f.wasConvertResultCalled()); - assertFalse("unexpected waitOperationResult() called", f.wasWaitOperationResultCalled()); - assertTrue("expected postOperationResult() to be called", f.wasPostOperationResultCalled()); + assertTrue(f.wasConvertResultCalled(), "expected convertResult() to be called"); + assertFalse(f.wasWaitOperationResultCalled(), "unexpected waitOperationResult() called"); + assertTrue(f.wasPostOperationResultCalled(), "expected postOperationResult() to be called"); } /** @@ -150,10 +145,10 @@ public void testWithoutProcId() throws Exception { TestFuture f = new TestFuture(admin, null); f.get(1, TimeUnit.MINUTES); - assertFalse("unexpected getProcedureResult() called", f.wasGetProcedureResultCalled()); - assertFalse("unexpected convertResult() called", f.wasConvertResultCalled()); - assertTrue("expected waitOperationResult() to be called", f.wasWaitOperationResultCalled()); - assertTrue("expected postOperationResult() to be called", f.wasPostOperationResultCalled()); + assertFalse(f.wasGetProcedureResultCalled(), "unexpected getProcedureResult() called"); + assertFalse(f.wasConvertResultCalled(), "unexpected convertResult() called"); + assertTrue(f.wasWaitOperationResultCalled(), "expected waitOperationResult() to be called"); + assertTrue(f.wasPostOperationResultCalled(), "expected postOperationResult() to be called"); } /** @@ -176,9 +171,9 @@ public void testOnServerWithNoProcedureSupport() throws Exception { }; f.get(1, TimeUnit.MINUTES); - assertTrue("expected getProcedureResult() to be called", f.wasGetProcedureResultCalled()); - assertFalse("unexpected convertResult() called", f.wasConvertResultCalled()); - assertTrue("expected waitOperationResult() to be called", f.wasWaitOperationResultCalled()); - assertTrue("expected postOperationResult() to be called", f.wasPostOperationResultCalled()); + assertTrue(f.wasGetProcedureResultCalled(), "expected getProcedureResult() to be called"); + assertFalse(f.wasConvertResultCalled(), "unexpected convertResult() called"); + assertTrue(f.wasWaitOperationResultCalled(), "expected waitOperationResult() to be called"); + assertTrue(f.wasPostOperationResultCalled(), "expected postOperationResult() to be called"); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java index 1e533633c767..507c7e98caa7 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPutDotHas.java @@ -17,26 +17,23 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) /** * Addresses HBASE-6047 We test put.has call with all of its polymorphic magic */ +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestPutDotHas { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestPutDotHas.class); - public static final byte[] ROW_01 = Bytes.toBytes("row-01"); public static final byte[] QUALIFIER_01 = Bytes.toBytes("qualifier-01"); public static final byte[] VALUE_01 = Bytes.toBytes("value-01"); @@ -44,39 +41,39 @@ public class TestPutDotHas { public static final long TS = 1234567L; public Put put = new Put(ROW_01); - @Before + @BeforeEach public void setUp() { put.addColumn(FAMILY_01, QUALIFIER_01, TS, VALUE_01); } @Test public void testHasIgnoreValueIgnoreTS() { - Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01)); - Assert.assertFalse(put.has(QUALIFIER_01, FAMILY_01)); + assertTrue(put.has(FAMILY_01, QUALIFIER_01)); + assertFalse(put.has(QUALIFIER_01, FAMILY_01)); } @Test public void testHasIgnoreValue() { - Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, TS)); - Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS + 1)); + assertTrue(put.has(FAMILY_01, QUALIFIER_01, TS)); + assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS + 1)); } @Test public void testHasIgnoreTS() { - Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, VALUE_01)); - Assert.assertFalse(put.has(FAMILY_01, VALUE_01, QUALIFIER_01)); + assertTrue(put.has(FAMILY_01, QUALIFIER_01, VALUE_01)); + assertFalse(put.has(FAMILY_01, VALUE_01, QUALIFIER_01)); } @Test public void testHas() { - Assert.assertTrue(put.has(FAMILY_01, QUALIFIER_01, TS, VALUE_01)); + assertTrue(put.has(FAMILY_01, QUALIFIER_01, TS, VALUE_01)); // Bad TS - Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS + 1, VALUE_01)); + assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS + 1, VALUE_01)); // Bad Value - Assert.assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS, QUALIFIER_01)); + assertFalse(put.has(FAMILY_01, QUALIFIER_01, TS, QUALIFIER_01)); // Bad Family - Assert.assertFalse(put.has(QUALIFIER_01, QUALIFIER_01, TS, VALUE_01)); + assertFalse(put.has(QUALIFIER_01, QUALIFIER_01, TS, VALUE_01)); // Bad Qual - Assert.assertFalse(put.has(FAMILY_01, FAMILY_01, TS, VALUE_01)); + assertFalse(put.has(FAMILY_01, FAMILY_01, TS, VALUE_01)); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPutWriteToWal.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPutWriteToWal.java index f57145cdb1b2..dcff979dfdea 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPutWriteToWal.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPutWriteToWal.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.client; -import org.junit.Before; +import org.junit.jupiter.api.BeforeEach; public class TestPutWriteToWal { - @Before + @BeforeEach public void setUp() throws Exception { } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionInfoBuilder.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionInfoBuilder.java index f74b79a0672e..b70a6b25ff15 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionInfoBuilder.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionInfoBuilder.java @@ -17,43 +17,42 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNameTestRule; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.MD5Hash; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestRegionInfoBuilder { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionInfoBuilder.class); + private TableName name; - @Rule - public TableNameTestRule name = new TableNameTestRule(); + @BeforeEach + public void setup(TestInfo testInfo) { + this.name = TableName.valueOf(testInfo.getTestMethod().get().getName()); + } @Test public void testBuilder() { @@ -92,7 +91,7 @@ public void testPb() throws DeserializationException { @Test public void testCreateRegionInfoName() throws Exception { - final TableName tn = name.getTableName(); + final TableName tn = name; String startKey = "startkey"; final byte[] sk = Bytes.toBytes(startKey); String id = "id"; @@ -112,7 +111,7 @@ public void testCreateRegionInfoName() throws Exception { @Test public void testContainsRange() { - TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(name.getTableName()).build(); + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(name).build(); RegionInfo ri = RegionInfoBuilder.newBuilder(tableDesc.getTableName()) .setStartKey(Bytes.toBytes("a")).setEndKey(Bytes.toBytes("g")).build(); // Single row range at start of region @@ -173,7 +172,7 @@ public void testContainsRangeForMetaTable() { @Test public void testLastRegionCompare() { - TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(name.getTableName()).build(); + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(name).build(); RegionInfo rip = RegionInfoBuilder.newBuilder(tableDesc.getTableName()) .setStartKey(Bytes.toBytes("a")).setEndKey(new byte[0]).build(); RegionInfo ric = RegionInfoBuilder.newBuilder(tableDesc.getTableName()) @@ -188,7 +187,7 @@ public void testMetaTables() { @Test public void testComparator() { - final TableName tableName = name.getTableName(); + final TableName tableName = name; byte[] empty = new byte[0]; RegionInfo older = RegionInfoBuilder.newBuilder(tableName).setStartKey(empty).setEndKey(empty) .setSplit(false).setRegionId(0L).build(); @@ -202,7 +201,7 @@ public void testComparator() { @Test public void testRegionNameForRegionReplicas() throws Exception { - final TableName tn = name.getTableName(); + final TableName tn = name; String startKey = "startkey"; final byte[] sk = Bytes.toBytes(startKey); String id = "id"; @@ -231,7 +230,7 @@ public void testRegionNameForRegionReplicas() throws Exception { @Test public void testParseName() throws IOException { - final TableName tableName = name.getTableName(); + final TableName tableName = name; byte[] startKey = Bytes.toBytes("startKey"); long regionId = EnvironmentEdgeManager.currentTime(); int replicaId = 42; @@ -240,26 +239,25 @@ public void testParseName() throws IOException { byte[] regionName = RegionInfo.createRegionName(tableName, startKey, regionId, false); byte[][] fields = RegionInfo.parseRegionName(regionName); - assertArrayEquals(Bytes.toString(fields[0]), tableName.getName(), fields[0]); - assertArrayEquals(Bytes.toString(fields[1]), startKey, fields[1]); - assertArrayEquals(Bytes.toString(fields[2]), Bytes.toBytes(Long.toString(regionId)), fields[2]); + assertArrayEquals(tableName.getName(), fields[0], Bytes.toString(fields[0])); + assertArrayEquals(startKey, fields[1], Bytes.toString(fields[1])); + assertArrayEquals(Bytes.toBytes(Long.toString(regionId)), fields[2], Bytes.toString(fields[2])); assertEquals(3, fields.length); // test with replicaId regionName = RegionInfo.createRegionName(tableName, startKey, regionId, replicaId, false); fields = RegionInfo.parseRegionName(regionName); - assertArrayEquals(Bytes.toString(fields[0]), tableName.getName(), fields[0]); - assertArrayEquals(Bytes.toString(fields[1]), startKey, fields[1]); - assertArrayEquals(Bytes.toString(fields[2]), Bytes.toBytes(Long.toString(regionId)), fields[2]); - assertArrayEquals(Bytes.toString(fields[3]), - Bytes.toBytes(String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)), fields[3]); + assertArrayEquals(tableName.getName(), fields[0], Bytes.toString(fields[0])); + assertArrayEquals(startKey, fields[1], Bytes.toString(fields[1])); + assertArrayEquals(Bytes.toBytes(Long.toString(regionId)), fields[2], Bytes.toString(fields[2])); + assertArrayEquals(Bytes.toBytes(String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)), + fields[3], Bytes.toString(fields[3])); } @Test public void testConvert() { - final TableName tableName = - TableName.valueOf("ns1:" + name.getTableName().getQualifierAsString()); + final TableName tableName = TableName.valueOf("ns1:" + name.getQualifierAsString()); byte[] startKey = Bytes.toBytes("startKey"); byte[] endKey = Bytes.toBytes("endKey"); boolean split = false; @@ -302,7 +300,7 @@ private void assertRegionNameEquals(RegionInfo expected, RegionInfo actual) { @Test public void testNewBuilderWithRegionInfo() { - RegionInfo ri = RegionInfoBuilder.newBuilder(name.getTableName()).build(); + RegionInfo ri = RegionInfoBuilder.newBuilder(name).build(); assertEquals(ri, RegionInfoBuilder.newBuilder(ri).build()); // make sure that the region name and encoded name are changed, see HBASE-24500 for more diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionInfoDisplay.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionInfoDisplay.java index 678eb28ce2ed..4b13dae6019f 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionInfoDisplay.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionInfoDisplay.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.IOException; import java.nio.charset.StandardCharsets; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.RegionState; @@ -28,40 +32,32 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestRegionInfoDisplay { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionInfoDisplay.class); - - @Rule - public TestName name = new TestName(); - @Test - public void testRegionDetailsForDisplay() throws IOException { + public void testRegionDetailsForDisplay(TestInfo testInfo) throws IOException { + String name = testInfo.getTestMethod().get().getName(); byte[] startKey = new byte[] { 0x01, 0x01, 0x02, 0x03 }; byte[] endKey = new byte[] { 0x01, 0x01, 0x02, 0x04 }; Configuration conf = new Configuration(); conf.setBoolean("hbase.display.keys", false); - RegionInfo ri = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setStartKey(startKey).setEndKey(endKey).build(); + RegionInfo ri = RegionInfoBuilder.newBuilder(TableName.valueOf(name)).setStartKey(startKey) + .setEndKey(endKey).build(); checkEquality(ri, conf); // check HRIs with non-default replicaId - ri = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).setStartKey(startKey) - .setEndKey(endKey).setSplit(false).setRegionId(EnvironmentEdgeManager.currentTime()) - .setReplicaId(1).build(); + ri = + RegionInfoBuilder.newBuilder(TableName.valueOf(name)).setStartKey(startKey).setEndKey(endKey) + .setSplit(false).setRegionId(EnvironmentEdgeManager.currentTime()).setReplicaId(1).build(); checkEquality(ri, conf); - Assert.assertArrayEquals(RegionInfoDisplay.HIDDEN_END_KEY, + assertArrayEquals(RegionInfoDisplay.HIDDEN_END_KEY, RegionInfoDisplay.getEndKeyForDisplay(ri, conf)); - Assert.assertArrayEquals(RegionInfoDisplay.HIDDEN_START_KEY, + assertArrayEquals(RegionInfoDisplay.HIDDEN_START_KEY, RegionInfoDisplay.getStartKeyForDisplay(ri, conf)); RegionState state = RegionState.createForTesting(convert(ri), RegionState.State.OPEN); @@ -71,9 +67,9 @@ public void testRegionDetailsForDisplay() throws IOException { checkDescriptiveNameEquality(descriptiveNameForDisplay, originalDescriptive, startKey); conf.setBoolean("hbase.display.keys", true); - Assert.assertArrayEquals(endKey, RegionInfoDisplay.getEndKeyForDisplay(ri, conf)); - Assert.assertArrayEquals(startKey, RegionInfoDisplay.getStartKeyForDisplay(ri, conf)); - Assert.assertEquals(originalDescriptive, + assertArrayEquals(endKey, RegionInfoDisplay.getEndKeyForDisplay(ri, conf)); + assertArrayEquals(startKey, RegionInfoDisplay.getStartKeyForDisplay(ri, conf)); + assertEquals(originalDescriptive, RegionInfoDisplay.getDescriptiveNameFromRegionStateForDisplay(state, conf)); } @@ -88,8 +84,8 @@ private void checkDescriptiveNameEquality(String descriptiveNameForDisplay, Stri String firstPartOrig = origDesc.substring(0, origDesc.indexOf(Bytes.toStringBinary(startKey))); String secondPartOrig = origDesc.substring( origDesc.indexOf(Bytes.toStringBinary(startKey)) + Bytes.toStringBinary(startKey).length()); - Assert.assertTrue(firstPart.equals(firstPartOrig)); - Assert.assertTrue(secondPart.equals(secondPartOrig)); + assertTrue(firstPart.equals(firstPartOrig)); + assertTrue(secondPart.equals(secondPartOrig)); } private void checkEquality(RegionInfo ri, Configuration conf) throws IOException { @@ -106,12 +102,12 @@ private void checkEquality(RegionInfo ri, Configuration conf) throws IOException if (i != 1) { System.out.println("" + i + " " + Bytes.toString(regionNameParts[i]) + " " + Bytes.toString(modifiedRegionNameParts[i])); - Assert.assertArrayEquals(regionNameParts[i], modifiedRegionNameParts[i]); + assertArrayEquals(regionNameParts[i], modifiedRegionNameParts[i]); } else { System.out.println("" + i + " " + Bytes.toString(regionNameParts[i]) + " " + Bytes.toString(modifiedRegionNameParts[i])); - Assert.assertNotEquals(regionNameParts[i], modifiedRegionNameParts[i]); - Assert.assertArrayEquals(modifiedRegionNameParts[1], + assertNotEquals(regionNameParts[i], modifiedRegionNameParts[i]); + assertArrayEquals(modifiedRegionNameParts[1], RegionInfoDisplay.getStartKeyForDisplay(ri, conf)); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocatorTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocatorTracing.java index 1e9860b283b4..8c22660b8268 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocatorTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegionLocatorTracing.java @@ -33,38 +33,34 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestRegionLocatorTracing extends TestTracingBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionLocatorTracing.class); ConnectionImplementation conn; @Override - @Before + @BeforeEach public void setUp() throws Exception { super.setUp(); conn = new ConnectionImplementation(conf, null, UserProvider.instantiate(conf).getCurrent(), Collections.emptyMap()); } - @After + @AfterEach public void tearDown() throws IOException { Closeables.close(conn, true); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegistryEndpointsRefresher.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegistryEndpointsRefresher.java index 3d6fe1563b8c..7605515cf1ba 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegistryEndpointsRefresher.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRegistryEndpointsRefresher.java @@ -17,35 +17,30 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.util.concurrent.Uninterruptibles; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestRegistryEndpointsRefresher { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegistryEndpointsRefresher.class); - private static final String INITIAL_DELAY_SECS_CONFIG_NAME = "hbase.test.registry.initial.delay.secs"; private static final String INTERVAL_SECS_CONFIG_NAME = @@ -58,14 +53,14 @@ public class TestRegistryEndpointsRefresher { private AtomicInteger refreshCallCounter; private CopyOnWriteArrayList callTimestamps; - @Before + @BeforeEach public void setUp() { conf = HBaseConfiguration.create(); refreshCallCounter = new AtomicInteger(0); callTimestamps = new CopyOnWriteArrayList<>(); } - @After + @AfterEach public void tearDown() { if (refresher != null) { refresher.stop(); @@ -123,13 +118,13 @@ public void testDurationBetweenRefreshes() { } // Overall wait time is 10000 ms, so the number of requests should be <=10 // Actual calls to refresh should be much lower than the refresh count. - assertTrue(String.valueOf(refreshCallCounter.get()), refreshCallCounter.get() <= 20); + assertTrue(refreshCallCounter.get() <= 20, String.valueOf(refreshCallCounter.get())); assertTrue(callTimestamps.size() > 0); // Verify that the delta between subsequent refresh is at least 1sec as configured. for (int i = 1; i < callTimestamps.size() - 1; i++) { long delta = callTimestamps.get(i) - callTimestamps.get(i - 1); // Few ms cushion to account for any env jitter. - assertTrue(callTimestamps.toString(), delta > 990); + assertTrue(delta > 990, callTimestamps.toString()); } } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestResultStatsUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestResultStatsUtil.java index 7ee074b84b9f..384baa6f745e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestResultStatsUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestResultStatsUtil.java @@ -17,23 +17,19 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.backoff.ServerStatistics; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestResultStatsUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestResultStatsUtil.class); private static final RegionLoadStats regionLoadStats = new RegionLoadStats(100, 10, 90); private static final byte[] regionName = { 80 }; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRetriesExhaustedWithDetailsException.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRetriesExhaustedWithDetailsException.java index f380be0048f3..e5c18cad8fb3 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRetriesExhaustedWithDetailsException.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRetriesExhaustedWithDetailsException.java @@ -17,37 +17,27 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.RegionTooBusyException; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.Mockito; -@Category({ SmallTests.class }) +@Tag(SmallTests.TAG) public class TestRetriesExhaustedWithDetailsException { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRetriesExhaustedWithDetailsException.class); - - @Rule - public TestName name = new TestName(); - /** * Assert that a RetriesExhaustedException that has RegionTooBusyException outputs region name. */ @Test - public void testRegionTooBusyException() { + public void testRegionTooBusyException(TestInfo testInfo) { + final String regionName = testInfo.getTestMethod().get().getName(); List ts = new ArrayList<>(1); - final String regionName = this.name.getMethodName(); ts.add(new RegionTooBusyException(regionName)); List rows = new ArrayList<>(1); rows.add(Mockito.mock(Row.class)); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestReversedScannerCallable.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestReversedScannerCallable.java index e9e1d7efa830..df87f5e22092 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestReversedScannerCallable.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestReversedScannerCallable.java @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -26,7 +27,6 @@ import java.io.IOException; import java.util.Collections; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; @@ -38,22 +38,18 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@RunWith(MockitoJUnitRunner.class) -@Category({ ClientTests.class, SmallTests.class }) +@ExtendWith(MockitoExtension.class) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestReversedScannerCallable { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReversedScannerCallable.class); - private static final TableName TABLE_NAME = TableName.valueOf("TestReversedScannerCallable"); private static final String HOSTNAME = "localhost"; @@ -70,13 +66,13 @@ public class TestReversedScannerCallable { @Mock private HRegionLocation regionLocation; - @Before + @BeforeEach public void setUp() throws Exception { when(connection.getConfiguration()).thenReturn(new Configuration()); - when(regionLocations.size()).thenReturn(1); - when(regionLocations.getRegionLocation(0)).thenReturn(regionLocation); - when(regionLocation.getHostname()).thenReturn(HOSTNAME); - when(regionLocation.getServerName()).thenReturn(SERVERNAME); + lenient().when(regionLocations.size()).thenReturn(1); + lenient().when(regionLocations.getRegionLocation(0)).thenReturn(regionLocation); + lenient().when(regionLocation.getHostname()).thenReturn(HOSTNAME); + lenient().when(regionLocation.getServerName()).thenReturn(SERVERNAME); } @Test diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRowComparator.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRowComparator.java index 9b2c598eaaae..95a1f79abbc9 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRowComparator.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRowComparator.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.Collections; @@ -27,21 +27,16 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestRowComparator { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRowComparator.class); - private static final List DEFAULT_ROWS = IntStream.range(1, 9).mapToObj(String::valueOf) .map(Bytes::toBytes).collect(Collectors.toList()); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java index 52e060745e6e..cdc6336ecaf8 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; import java.net.SocketAddress; @@ -32,7 +32,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.RpcClient; @@ -41,12 +40,11 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FutureUtils; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,13 +60,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetClusterIdResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetConnectionRegistryResponse; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestRpcBasedRegistryHedgedReads { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRpcBasedRegistryHedgedReads.class); - private static final Logger LOG = LoggerFactory.getLogger(TestRpcBasedRegistryHedgedReads.class); private static final String HEDGED_REQS_FANOUT_CONFIG_NAME = "hbase.test.hedged.reqs.fanout"; @@ -184,7 +179,7 @@ public String getConnectionString() { }; } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() { Configuration conf = UTIL.getConfiguration(); conf.setClass(RpcClientFactory.CUSTOM_RPC_CLIENT_IMPL_CONF_KEY, RpcClientImpl.class, @@ -198,12 +193,12 @@ public static void setUpBeforeClass() { .collect(Collectors.toSet()); } - @AfterClass + @AfterAll public static void tearDownAfterClass() { EXECUTOR.shutdownNow(); } - @Before + @BeforeEach public void setUp() { CALLED.set(0); BAD_RESP_INDEX = -1; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcRetryingCallerImpl.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcRetryingCallerImpl.java index a01771900351..d80f1d5a766e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcRetryingCallerImpl.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcRetryingCallerImpl.java @@ -17,28 +17,23 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.CallQueueTooBigException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseServerException; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestRpcRetryingCallerImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRpcRetryingCallerImpl.class); - @Test public void itUsesSpecialPauseForCQTBE() throws Exception { itUsesSpecialPauseForServerOverloaded(CallQueueTooBigException.class); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java index 05d0fc865f7c..4bc1ad2c8b15 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java @@ -17,16 +17,16 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.Arrays; import java.util.Set; import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Scan.ReadType; import org.apache.hadoop.hbase.filter.FilterList; @@ -35,19 +35,16 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; // TODO: cover more test cases -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestScan { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestScan.class); @Test public void testAttributesSerialization() throws IOException { @@ -60,11 +57,11 @@ public void testAttributesSerialization() throws IOException { Scan scan2 = ProtobufUtil.toScan(scanProto); - Assert.assertNull(scan2.getAttribute("absent")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan2.getAttribute("attribute1"))); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan2.getAttribute("attribute2"))); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), scan2.getAttribute("attribute3"))); - Assert.assertEquals(3, scan2.getAttributesMap().size()); + assertNull(scan2.getAttribute("absent")); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan2.getAttribute("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan2.getAttribute("attribute2"))); + assertTrue(Arrays.equals(Bytes.toBytes("value3"), scan2.getAttribute("attribute3"))); + assertEquals(3, scan2.getAttributesMap().size()); } @Test @@ -108,51 +105,48 @@ public void testGetToScan() throws Exception { @Test public void testScanAttributes() { Scan scan = new Scan(); - Assert.assertTrue(scan.getAttributesMap().isEmpty()); - Assert.assertNull(scan.getAttribute("absent")); + assertTrue(scan.getAttributesMap().isEmpty()); + assertNull(scan.getAttribute("absent")); scan.setAttribute("absent", null); - Assert.assertTrue(scan.getAttributesMap().isEmpty()); - Assert.assertNull(scan.getAttribute("absent")); + assertTrue(scan.getAttributesMap().isEmpty()); + assertNull(scan.getAttribute("absent")); // adding attribute scan.setAttribute("attribute1", Bytes.toBytes("value1")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan.getAttribute("attribute1"))); - Assert.assertEquals(1, scan.getAttributesMap().size()); - Assert.assertTrue( - Arrays.equals(Bytes.toBytes("value1"), scan.getAttributesMap().get("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan.getAttribute("attribute1"))); + assertEquals(1, scan.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan.getAttributesMap().get("attribute1"))); // overriding attribute value scan.setAttribute("attribute1", Bytes.toBytes("value12")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value12"), scan.getAttribute("attribute1"))); - Assert.assertEquals(1, scan.getAttributesMap().size()); - Assert.assertTrue( - Arrays.equals(Bytes.toBytes("value12"), scan.getAttributesMap().get("attribute1"))); + assertTrue(Arrays.equals(Bytes.toBytes("value12"), scan.getAttribute("attribute1"))); + assertEquals(1, scan.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value12"), scan.getAttributesMap().get("attribute1"))); // adding another attribute scan.setAttribute("attribute2", Bytes.toBytes("value2")); - Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan.getAttribute("attribute2"))); - Assert.assertEquals(2, scan.getAttributesMap().size()); - Assert.assertTrue( - Arrays.equals(Bytes.toBytes("value2"), scan.getAttributesMap().get("attribute2"))); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan.getAttribute("attribute2"))); + assertEquals(2, scan.getAttributesMap().size()); + assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan.getAttributesMap().get("attribute2"))); // removing attribute scan.setAttribute("attribute2", null); - Assert.assertNull(scan.getAttribute("attribute2")); - Assert.assertEquals(1, scan.getAttributesMap().size()); - Assert.assertNull(scan.getAttributesMap().get("attribute2")); + assertNull(scan.getAttribute("attribute2")); + assertEquals(1, scan.getAttributesMap().size()); + assertNull(scan.getAttributesMap().get("attribute2")); // removing non-existed attribute scan.setAttribute("attribute2", null); - Assert.assertNull(scan.getAttribute("attribute2")); - Assert.assertEquals(1, scan.getAttributesMap().size()); - Assert.assertNull(scan.getAttributesMap().get("attribute2")); + assertNull(scan.getAttribute("attribute2")); + assertEquals(1, scan.getAttributesMap().size()); + assertNull(scan.getAttributesMap().get("attribute2")); // removing another attribute scan.setAttribute("attribute1", null); - Assert.assertNull(scan.getAttribute("attribute1")); - Assert.assertTrue(scan.getAttributesMap().isEmpty()); - Assert.assertNull(scan.getAttributesMap().get("attribute1")); + assertNull(scan.getAttribute("attribute1")); + assertTrue(scan.getAttributesMap().isEmpty()); + assertNull(scan.getAttributesMap().get("attribute1")); } @Test @@ -161,7 +155,7 @@ public void testNullQualifier() { byte[] family = Bytes.toBytes("family"); scan.addColumn(family, null); Set qualifiers = scan.getFamilyMap().get(family); - Assert.assertEquals(1, qualifiers.size()); + assertEquals(1, qualifiers.size()); } @Test @@ -257,8 +251,8 @@ public void testScanCopyConstructor() throws Exception { assertEquals(scan.getTimeRange(), scanCopy.getTimeRange()); assertEquals(scan.isQueryMetricsEnabled(), scanCopy.isQueryMetricsEnabled()); - assertTrue("Make sure copy constructor adds all the fields in the copied object", - EqualsBuilder.reflectionEquals(scan, scanCopy)); + assertTrue(EqualsBuilder.reflectionEquals(scan, scanCopy), + "Make sure copy constructor adds all the fields in the copied object"); } @Test diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScannerCallable.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScannerCallable.java index e675b06b574b..cf471bfda1ce 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScannerCallable.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScannerCallable.java @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -25,7 +26,6 @@ import java.io.IOException; import java.util.Collections; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; @@ -35,20 +35,17 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@RunWith(MockitoJUnitRunner.class) -@Category({ ClientTests.class, SmallTests.class }) +@ExtendWith(MockitoExtension.class) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestScannerCallable { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestScannerCallable.class); private static final TableName TABLE_NAME = TableName.valueOf("TestScannerCallable"); @@ -66,13 +63,13 @@ public class TestScannerCallable { @Mock private HRegionLocation regionLocation; - @Before + @BeforeEach public void setUp() throws Exception { when(connection.getConfiguration()).thenReturn(new Configuration()); - when(regionLocations.size()).thenReturn(1); - when(regionLocations.getRegionLocation(0)).thenReturn(regionLocation); - when(regionLocation.getHostname()).thenReturn(HOSTNAME); - when(regionLocation.getServerName()).thenReturn(SERVERNAME); + lenient().when(regionLocations.size()).thenReturn(1); + lenient().when(regionLocations.getRegionLocation(0)).thenReturn(regionLocation); + lenient().when(regionLocation.getHostname()).thenReturn(HOSTNAME); + lenient().when(regionLocation.getServerName()).thenReturn(SERVERNAME); } @Test diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSimpleRequestController.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSimpleRequestController.java index 6cb9960b96b0..7458e0acc901 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSimpleRequestController.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSimpleRequestController.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.io.InterruptedIOException; @@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; @@ -44,18 +43,13 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestSimpleRequestController { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSimpleRequestController.class); - private static final TableName DUMMY_TABLE = TableName.valueOf("DUMMY_TABLE"); private static final byte[] DUMMY_BYTES_1 = Bytes.toBytes("DUMMY_BYTES_1"); private static final byte[] DUMMY_BYTES_2 = Bytes.toBytes("DUMMY_BYTES_2"); @@ -362,7 +356,7 @@ public void testWaitForMaximumCurrentTasks() throws Exception { barrier.await(); controller.waitForMaximumCurrentTasks(max.get(), 123, 1, null); } catch (InterruptedIOException | InterruptedException | BrokenBarrierException e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } }; // First test that our runnable thread only exits when tasks is zero. diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorBuilder.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorBuilder.java index 8a653ddd8ab7..01f0a8cae927 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorBuilder.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorBuilder.java @@ -17,16 +17,15 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.regex.Pattern; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.exceptions.HBaseException; @@ -34,33 +33,33 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Test setting values in the descriptor. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestTableDescriptorBuilder { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableDescriptorBuilder.class); private static final Logger LOG = LoggerFactory.getLogger(TestTableDescriptorBuilder.class); - @Rule - public TestName name = new TestName(); + public String name; - @Test(expected = IOException.class) + @org.junit.jupiter.api.BeforeEach + public void setupTestName(TestInfo testInfo) { + name = testInfo.getTestMethod().get().getName(); + } + + @Test public void testAddCoprocessorTwice() throws IOException { String cpName = "a.b.c.d"; - TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).setCoprocessor(cpName) - .setCoprocessor(cpName).build(); + assertThrows(IOException.class, () -> TableDescriptorBuilder + .newBuilder(TableName.META_TABLE_NAME).setCoprocessor(cpName).setCoprocessor(cpName).build()); } @Test @@ -87,12 +86,14 @@ public void testPb() throws DeserializationException, IOException { public void testGetSetRemoveCP() throws Exception { // simple CP String className = "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver"; - TableDescriptor desc = TableDescriptorBuilder - .newBuilder(TableName.valueOf(name.getMethodName())).setCoprocessor(className) // add and - // check that - // it is - // present - .build(); + TableDescriptor desc = + TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setCoprocessor(className) // add + // and + // check + // that + // it is + // present + .build(); assertTrue(desc.hasCoprocessor(className)); desc = TableDescriptorBuilder.newBuilder(desc).removeCoprocessor(className) // remove it and // check that it is @@ -107,8 +108,7 @@ public void testGetSetRemoveCP() throws Exception { */ @Test public void testSetListRemoveCP() throws Exception { - TableDescriptor desc = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); // Check that any coprocessor is present. assertTrue(desc.getCoprocessorDescriptors().isEmpty()); @@ -156,8 +156,7 @@ public void testSetListRemoveCP() throws Exception { @Test public void testRemoveNonExistingCoprocessor() throws Exception { String className = "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver"; - TableDescriptor desc = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); assertFalse(desc.hasCoprocessor(className)); desc = TableDescriptorBuilder.newBuilder(desc).removeCoprocessor(className).build(); assertFalse(desc.hasCoprocessor(className)); @@ -170,8 +169,8 @@ public void testRemoveNonExistingCoprocessor() throws Exception { public void testRemoveString() { byte[] key = Bytes.toBytes("Some"); byte[] value = Bytes.toBytes("value"); - TableDescriptor desc = TableDescriptorBuilder - .newBuilder(TableName.valueOf(name.getMethodName())).setValue(key, value).build(); + TableDescriptor desc = + TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setValue(key, value).build(); assertTrue(Bytes.equals(value, desc.getValue(key))); desc = TableDescriptorBuilder.newBuilder(desc).removeValue(key).build(); assertTrue(desc.getValue(key) == null); @@ -208,8 +207,8 @@ public void testIllegalTableNames() { public void testLegalTableNamesRegex() { for (String tn : legalTableNames) { TableName tName = TableName.valueOf(tn); - assertTrue("Testing: '" + tn + "'", - Pattern.matches(TableName.VALID_USER_TABLE_REGEX, tName.getNameAsString())); + assertTrue(Pattern.matches(TableName.VALID_USER_TABLE_REGEX, tName.getNameAsString()), + "Testing: '" + tn + "'"); } } @@ -226,18 +225,15 @@ public void testIllegalTableNamesRegex() { */ @Test public void testGetMaxFileSize() { - TableDescriptor desc = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); assertEquals(-1, desc.getMaxFileSize()); - desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setMaxFileSize(1111L).build(); + desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setMaxFileSize(1111L).build(); assertEquals(1111L, desc.getMaxFileSize()); } @Test public void testSetMaxFileSize() throws HBaseException { - TableDescriptorBuilder builder = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); String maxFileSize = "1073741824"; builder.setMaxFileSize(maxFileSize); @@ -266,18 +262,16 @@ public void testSetMaxFileSize() throws HBaseException { */ @Test public void testGetMemStoreFlushSize() { - TableDescriptor desc = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); assertEquals(-1, desc.getMemStoreFlushSize()); - desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setMemStoreFlushSize(1111L).build(); + desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setMemStoreFlushSize(1111L) + .build(); assertEquals(1111L, desc.getMemStoreFlushSize()); } @Test public void testSetMemStoreFlushSize() throws HBaseException { - TableDescriptorBuilder builder = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); String memstoreFlushSize = "1073741824"; builder.setMemStoreFlushSize(memstoreFlushSize); @@ -311,8 +305,8 @@ public void testModifyFamily() { byte[] familyName = Bytes.toBytes("cf"); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(1000).setDFSReplication((short) 3).build(); - TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(hcd).build(); + TableDescriptor htd = + TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setColumnFamily(hcd).build(); assertEquals(1000, htd.getColumnFamily(familyName).getBlocksize()); assertEquals(3, htd.getColumnFamily(familyName).getDFSReplication()); @@ -323,31 +317,33 @@ public void testModifyFamily() { assertEquals(1, htd.getColumnFamily(familyName).getDFSReplication()); } - @Test(expected = IllegalArgumentException.class) + @Test public void testModifyInexistentFamily() { byte[] familyName = Bytes.toBytes("cf"); - HColumnDescriptor hcd = new HColumnDescriptor(familyName); - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .modifyColumnFamily(hcd).build(); + assertThrows(IllegalArgumentException.class, + () -> TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) + .modifyColumnFamily(ColumnFamilyDescriptorBuilder.of(familyName)).build()); } - @Test(expected = IllegalArgumentException.class) + @Test public void testAddDuplicateFamilies() { byte[] familyName = Bytes.toBytes("cf"); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName).setBlocksize(1000).build(); - TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(hcd).build(); + TableDescriptor htd = + TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setColumnFamily(hcd).build(); assertEquals(1000, htd.getColumnFamily(familyName).getBlocksize()); hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName).setBlocksize(2000).build(); // add duplicate column - TableDescriptorBuilder.newBuilder(htd).setColumnFamily(hcd).build(); + ColumnFamilyDescriptor finalHcd = hcd; + assertThrows(IllegalArgumentException.class, + () -> TableDescriptorBuilder.newBuilder(htd).setColumnFamily(finalHcd).build()); } @Test public void testPriority() { - TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setPriority(42).build(); + TableDescriptor htd = + TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setPriority(42).build(); assertEquals(42, htd.getPriority()); } @@ -356,7 +352,7 @@ public void testStringCustomizedValues() throws HBaseException { byte[] familyName = Bytes.toBytes("cf"); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName).setBlocksize(131072).build(); - TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) + TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) .setColumnFamily(hcd).setDurability(Durability.ASYNC_WAL).build(); assertEquals( @@ -376,8 +372,7 @@ public void testStringCustomizedValues() throws HBaseException { @Test public void testSetEmptyValue() { - TableDescriptorBuilder builder = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); String testValue = "TestValue"; // test setValue builder.setValue(testValue, "2"); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorUtils.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorUtils.java index 8ba350b03b10..6231820e4aa4 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorUtils.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorUtils.java @@ -17,23 +17,18 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Arrays; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.TableDescriptorUtils.TableDescriptorDelta; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestTableDescriptorUtils { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableDescriptorUtils.class); @Test public void testDelta() { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableEnableDisableError.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableEnableDisableError.java index 4a49abaf2ccf..06de7d91a3b4 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableEnableDisableError.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableEnableDisableError.java @@ -17,39 +17,34 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import org.apache.hadoop.hbase.AsyncMetaTableAccessor; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FutureUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestTableEnableDisableError { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableEnableDisableError.class); - private MockedStatic mockedClientMetaTableAccessor; private AsyncAdmin admin; - @Before + @BeforeEach public void setUp() { mockedClientMetaTableAccessor = mockStatic(AsyncMetaTableAccessor.class); AsyncConnectionImpl conn = mock(AsyncConnectionImpl.class); @@ -64,7 +59,7 @@ public AsyncAdmin build() { admin = builder.build(); } - @After + @AfterEach public void tearDown() { mockedClientMetaTableAccessor.closeOnDemand(); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableRpcPriority.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableRpcPriority.java index 3bdd83effebf..ec30bbfd53e9 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableRpcPriority.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableRpcPriority.java @@ -21,10 +21,10 @@ import static org.apache.hadoop.hbase.HConstants.NORMAL_QOS; import static org.apache.hadoop.hbase.HConstants.SYSTEMTABLE_QOS; import static org.apache.hadoop.hbase.NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.argThat; import static org.mockito.Mockito.atLeast; @@ -43,7 +43,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLocations; @@ -55,12 +54,10 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -74,21 +71,18 @@ * Test that correct rpc priority is sent to server from blocking Table calls. Currently only * implements checks for scans, but more could be added here. */ -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestTableRpcPriority { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableRpcPriority.class); - - @Rule - public TestName name = new TestName(); + private String methodName; private ClientProtos.ClientService.BlockingInterface stub; private Connection conn; - @Before - public void setUp() throws IOException, ServiceException { + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException, ServiceException { + methodName = testInfo.getTestMethod().get().getName(); stub = mock(ClientProtos.ClientService.BlockingInterface.class); Configuration conf = HBaseConfiguration.create(); @@ -124,7 +118,7 @@ public RegionLocations locateRegion(TableName tableName, byte[] row, boolean use @Test public void testScan() throws Exception { mockScan(19); - testForTable(TableName.valueOf(name.getMethodName()), Optional.of(19)); + testForTable(TableName.valueOf(methodName), Optional.of(19)); } /** @@ -134,20 +128,19 @@ public void testScan() throws Exception { @Test public void testScanSuperHighPriority() throws Exception { mockScan(1000); - testForTable(TableName.valueOf(name.getMethodName()), Optional.of(1000)); + testForTable(TableName.valueOf(methodName), Optional.of(1000)); } @Test public void testScanNormalTable() throws Exception { mockScan(NORMAL_QOS); - testForTable(TableName.valueOf(name.getMethodName()), Optional.of(NORMAL_QOS)); + testForTable(TableName.valueOf(methodName), Optional.of(NORMAL_QOS)); } @Test public void testScanSystemTable() throws Exception { mockScan(SYSTEMTABLE_QOS); - testForTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, name.getMethodName()), - Optional.empty()); + testForTable(TableName.valueOf(SYSTEM_NAMESPACE_NAME_STR, methodName), Optional.empty()); } @Test @@ -190,8 +183,8 @@ public ClientProtos.ScanResponse answer(InvocationOnMock invocation) throws Thro @Override public ClientProtos.ScanResponse answer(InvocationOnMock invocation) throws Throwable { ClientProtos.ScanRequest req = invocation.getArgument(1); - assertFalse("close scanner should not come in with scan priority " + scanPriority, - req.hasCloseScanner() && req.getCloseScanner()); + assertFalse(req.hasCloseScanner() && req.getCloseScanner(), + "close scanner should not come in with scan priority " + scanPriority); ClientProtos.ScanResponse.Builder builder = ClientProtos.ScanResponse.newBuilder(); if (!req.hasScannerId()) { @@ -214,10 +207,10 @@ public ClientProtos.ScanResponse answer(InvocationOnMock invocation) throws Thro @Override public ClientProtos.ScanResponse answer(InvocationOnMock invocation) throws Throwable { ClientProtos.ScanRequest req = invocation.getArgument(1); - assertTrue("close request should have scannerId", req.hasScannerId()); - assertEquals("close request's scannerId should match", scannerId, req.getScannerId()); - assertTrue("close request should have closerScanner set", - req.hasCloseScanner() && req.getCloseScanner()); + assertTrue(req.hasScannerId(), "close request should have scannerId"); + assertEquals(scannerId, req.getScannerId(), "close request's scannerId should match"); + assertTrue(req.hasCloseScanner() && req.getCloseScanner(), + "close request should have closerScanner set"); return ClientProtos.ScanResponse.getDefaultInstance(); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java index d1eefe72a03d..155f482781d9 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java @@ -21,12 +21,12 @@ import static org.apache.hadoop.hbase.client.trace.hamcrest.SpanDataMatchers.hasName; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.hasItem; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; import io.opentelemetry.sdk.trace.data.SpanData; import java.util.Objects; import java.util.Optional; @@ -43,8 +43,8 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.trace.HBaseSemanticAttributes; import org.hamcrest.Matcher; -import org.junit.Before; -import org.junit.ClassRule; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,10 +57,10 @@ public class TestTracingBase { protected Configuration conf; - @ClassRule - public static OpenTelemetryRule TRACE_RULE = OpenTelemetryRule.create(); + @RegisterExtension + public static final OpenTelemetryExtension TRACE_RULE = OpenTelemetryExtension.create(); - @Before + @BeforeEach public void setUp() throws Exception { conf = HBaseConfiguration.create(); conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/backoff/TestHBaseServerExceptionPauseManager.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/backoff/TestHBaseServerExceptionPauseManager.java index ee4ee47f1850..513f19d7af6f 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/backoff/TestHBaseServerExceptionPauseManager.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/backoff/TestHBaseServerExceptionPauseManager.java @@ -17,22 +17,21 @@ */ package org.apache.hadoop.hbase.client.backoff; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.OptionalLong; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseServerException; import org.apache.hadoop.hbase.quotas.RpcThrottlingException; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestHBaseServerExceptionPauseManager { private static final long WAIT_INTERVAL_MILLIS = 1L; @@ -47,10 +46,6 @@ public class TestHBaseServerExceptionPauseManager { private final Throwable OTHER_EXCEPTION = new RuntimeException(""); private final HBaseServerException SERVER_OVERLOADED_EXCEPTION = new HBaseServerException(true); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHBaseServerExceptionPauseManager.class); - @Test public void itSupportsRpcThrottlingNanosNoTimeout() { HBaseServerExceptionPauseManager pauseManager = diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/exceptions/TestClientExceptionsUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/exceptions/TestClientExceptionsUtil.java index 401d38d66cd0..417d807bae57 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/exceptions/TestClientExceptionsUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/exceptions/TestClientExceptionsUtil.java @@ -17,24 +17,20 @@ */ package org.apache.hadoop.hbase.exceptions; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; @SuppressWarnings("ThrowableInstanceNeverThrown") -@Category({ SmallTests.class, ClientTests.class }) +@Tag(SmallTests.TAG) +@Tag(ClientTests.TAG) public class TestClientExceptionsUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestClientExceptionsUtil.class); @Test public void testFindException() { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java index 721c258fb303..3ad0674bd833 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java @@ -17,30 +17,25 @@ */ package org.apache.hadoop.hbase.filter; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.ByteBuffer; import org.apache.hadoop.hbase.ByteBufferKeyValue; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestComparators { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestComparators.class); - @Test public void testCellFieldsCompare() throws Exception { byte[] r0 = Bytes.toBytes("row0"); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java index f1f1e37ff524..e42057057eb7 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java @@ -17,15 +17,14 @@ */ package org.apache.hadoop.hbase.filter; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.ByteBuffer; -import java.util.Collection; +import java.util.stream.Stream; import org.apache.hadoop.hbase.ByteBufferKeyValue; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.filter.KeyOnlyFilter.KeyOnlyByteBufferExtendedCell; @@ -33,30 +32,26 @@ import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -@Category({ MiscTests.class, SmallTests.class }) -@RunWith(Parameterized.class) -public class TestKeyOnlyFilter { +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.params.provider.Arguments; - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestKeyOnlyFilter.class); +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +@HBaseParameterizedTestTemplate +public class TestKeyOnlyFilter { - @Parameterized.Parameter public boolean lenAsVal; - @Parameters - public static Collection parameters() { - return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; + public TestKeyOnlyFilter(boolean lenAsVal) { + this.lenAsVal = lenAsVal; + } + + public static Stream parameters() { + return Stream.of(Arguments.of(true), Arguments.of(false)); } - @Test + @TestTemplate public void testKeyOnly() throws Exception { byte[] r = Bytes.toBytes("row1"); byte[] f = Bytes.toBytes("cf1"); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestLongComparator.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestLongComparator.java index ae5fb23161a7..ac16dfb5d176 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestLongComparator.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestLongComparator.java @@ -17,21 +17,16 @@ */ package org.apache.hadoop.hbase.filter; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.ByteBuffer; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestLongComparator { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLongComparator.class); private long[] values = { Long.MIN_VALUE, -10000000000L, -1000000L, 0L, 1000000L, 10000000000L, Long.MAX_VALUE }; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestCellBlockBuilder.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestCellBlockBuilder.java index fccea923635a..628866c7d451 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestCellBlockBuilder.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestCellBlockBuilder.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.ipc; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.nio.ByteBuffer; @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -41,24 +40,21 @@ import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.io.compress.GzipCodec; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestCellBlockBuilder { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCellBlockBuilder.class); private static final Logger LOG = LoggerFactory.getLogger(TestCellBlockBuilder.class); private CellBlockBuilder builder; - @Before + @BeforeEach public void before() { this.builder = new CellBlockBuilder(HBaseConfiguration.create()); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestConnectionId.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestConnectionId.java index da962cac0d3c..9402dd4ab319 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestConnectionId.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestConnectionId.java @@ -17,27 +17,23 @@ */ package org.apache.hadoop.hbase.ipc; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ SmallTests.class, ClientTests.class }) +@Tag(SmallTests.TAG) +@Tag(ClientTests.TAG) public class TestConnectionId { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestConnectionId.class); private Configuration testConfig = HBaseConfiguration.create(); private User testUser1 = diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestFailedServersLog.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestFailedServersLog.java index 0dafef0b7649..d939c3cb834a 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestFailedServersLog.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestFailedServersLog.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.ipc; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -27,32 +27,27 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestFailedServersLog { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFailedServersLog.class); - static final int TEST_PORT = 9999; private Address addr; private org.apache.logging.log4j.core.Appender mockAppender; - @Before + @BeforeEach public void setup() { mockAppender = mock(org.apache.logging.log4j.core.Appender.class); when(mockAppender.getName()).thenReturn("mockAppender"); @@ -62,7 +57,7 @@ public void setup() { } - @After + @AfterEach public void teardown() { ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager .getLogger(FailedServers.class)).removeAppender(mockAppender); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestHBaseRpcControllerImpl.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestHBaseRpcControllerImpl.java index a0b68646b145..b8a04fbbc302 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestHBaseRpcControllerImpl.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestHBaseRpcControllerImpl.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.ipc; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -26,19 +26,15 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScannable; import org.apache.hadoop.hbase.CellScanner; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestHBaseRpcControllerImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHBaseRpcControllerImpl.class); @Test public void testListOfCellScannerables() throws IOException { @@ -54,8 +50,8 @@ public void testListOfCellScannerables() throws IOException { for (; cellScanner.advance(); index++) { Cell cell = cellScanner.current(); byte[] indexBytes = Bytes.toBytes(index); - assertTrue("" + index, Bytes.equals(indexBytes, 0, indexBytes.length, cell.getValueArray(), - cell.getValueOffset(), cell.getValueLength())); + assertTrue(Bytes.equals(indexBytes, 0, indexBytes.length, cell.getValueArray(), + cell.getValueOffset(), cell.getValueLength()), "" + index); } assertEquals(count, index); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java index d0e4044b0456..75222ba5be4e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java @@ -19,8 +19,8 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.lang.reflect.Constructor; @@ -30,7 +30,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeoutException; import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil; import org.apache.hadoop.hbase.exceptions.TimeoutIOException; @@ -38,22 +37,18 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FutureUtils; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.io.netty.channel.DefaultEventLoop; import org.apache.hbase.thirdparty.io.netty.channel.EventLoop; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ExceptionResponse; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestIPCUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIPCUtil.class); - private static Throwable create(Class clazz) throws InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException { try { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcConnection.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcConnection.java index 73f3653a9026..ee4b50608f87 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcConnection.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcConnection.java @@ -19,51 +19,46 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestNettyRpcConnection { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestNettyRpcConnection.class); - private static final Logger LOG = LoggerFactory.getLogger(TestNettyRpcConnection.class); private static NettyRpcClient CLIENT; private static NettyRpcConnection CONN; - @BeforeClass + @BeforeAll public static void setUp() throws IOException { CLIENT = new NettyRpcClient(HBaseConfiguration.create()); CONN = new NettyRpcConnection(CLIENT, new ConnectionId(User.getCurrent(), "test", Address.fromParts("localhost", 1234))); } - @AfterClass + @AfterAll public static void tearDown() throws IOException { Closeables.close(CLIENT, true); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestRemoteWithExtrasException.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestRemoteWithExtrasException.java index dfa3450b74a6..4f8dbc225acc 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestRemoteWithExtrasException.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestRemoteWithExtrasException.java @@ -17,26 +17,21 @@ */ package org.apache.hadoop.hbase.ipc; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseServerException; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestRemoteWithExtrasException { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRemoteWithExtrasException.class); - /** * test verifies that we honor the inherent value of an exception for isServerOverloaded. We don't * want a false value passed into RemoteWithExtrasExceptions to override the inherent value of an diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientDeprecatedNameMapping.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientDeprecatedNameMapping.java index 48bd5498cd41..7f4fcd2d6785 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientDeprecatedNameMapping.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientDeprecatedNameMapping.java @@ -21,22 +21,17 @@ import static org.hamcrest.MatcherAssert.assertThat; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestRpcClientDeprecatedNameMapping { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRpcClientDeprecatedNameMapping.class); - @Test public void test() { Configuration conf = HBaseConfiguration.create(); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestTLSHandshadeFailure.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestTLSHandshadeFailure.java index 8aadce85651d..b37017c97e5e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestTLSHandshadeFailure.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestTLSHandshadeFailure.java @@ -20,7 +20,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.startsWith; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -34,7 +34,6 @@ import java.util.Random; import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.MetricsConnection.CallStats; @@ -43,11 +42,10 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.Logger; @@ -64,13 +62,10 @@ *

* See HBASE-27782 for more details. */ -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestTLSHandshadeFailure { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTLSHandshadeFailure.class); - private static final Logger LOG = LoggerFactory.getLogger(TestTLSHandshadeFailure.class); private final Configuration conf = HBaseConfiguration.create(); @@ -107,7 +102,7 @@ private void serve() { } } - @Before + @BeforeEach public void setUp() throws IOException { server = new ServerSocket(0); serverThread = new Thread(this::serve); @@ -124,7 +119,7 @@ public void setUp() throws IOException { .getLogger(BufferCallBeforeInitHandler.class)).addAppender(mockAppender); } - @After + @AfterEach public void tearDown() throws IOException { ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager .getLogger(BufferCallBeforeInitHandler.class)).removeAppender(mockAppender); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaFilter.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaFilter.java index 25010d190f65..30e736e35a3d 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaFilter.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaFilter.java @@ -17,19 +17,15 @@ */ package org.apache.hadoop.hbase.quotas; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BuilderStyleTest; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestQuotaFilter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestQuotaFilter.class); @Test public void testClassMethodsAreBuilderStyle() { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaGlobalsSettingsBypass.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaGlobalsSettingsBypass.java index a3c96eb45aa9..7579692bd645 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaGlobalsSettingsBypass.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaGlobalsSettingsBypass.java @@ -17,26 +17,20 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory.QuotaGlobalsSettingsBypass; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ SmallTests.class }) +@Tag(SmallTests.TAG) public class TestQuotaGlobalsSettingsBypass { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestQuotaGlobalsSettingsBypass.class); - @Test public void testMerge() throws IOException { QuotaGlobalsSettingsBypass orig = new QuotaGlobalsSettingsBypass("joe", null, null, null, true); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java index be659bc202d7..04823dd019bc 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java @@ -17,21 +17,20 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; @@ -45,11 +44,8 @@ /** * Test class for {@link QuotaSettingsFactory}. */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestQuotaSettingsFactory { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestQuotaSettingsFactory.class); @Test public void testAllQuotasAddedToList() { @@ -80,59 +76,61 @@ public void testAllQuotasAddedToList() { ThrottleSettings throttleSettings = (ThrottleSettings) setting; switch (throttleSettings.getThrottleType()) { case READ_NUMBER: - assertFalse("Should not have multiple read quotas", seenRead); + assertFalse(seenRead, "Should not have multiple read quotas"); assertEquals(readLimit, throttleSettings.getSoftLimit()); assertEquals(TimeUnit.MINUTES, throttleSettings.getTimeUnit()); assertEquals(tn, throttleSettings.getTableName()); - assertNull("Username should be null", throttleSettings.getUserName()); - assertNull("Namespace should be null", throttleSettings.getNamespace()); - assertNull("RegionServer should be null", throttleSettings.getRegionServer()); + assertNull(throttleSettings.getUserName(), "Username should be null"); + assertNull(throttleSettings.getNamespace(), "Namespace should be null"); + assertNull(throttleSettings.getRegionServer(), "RegionServer should be null"); seenRead = true; break; case WRITE_NUMBER: - assertFalse("Should not have multiple write quotas", seenWrite); + assertFalse(seenWrite, "Should not have multiple write quotas"); assertEquals(writeLimit, throttleSettings.getSoftLimit()); assertEquals(TimeUnit.MINUTES, throttleSettings.getTimeUnit()); assertEquals(tn, throttleSettings.getTableName()); - assertNull("Username should be null", throttleSettings.getUserName()); - assertNull("Namespace should be null", throttleSettings.getNamespace()); - assertNull("RegionServer should be null", throttleSettings.getRegionServer()); + assertNull(throttleSettings.getUserName(), "Username should be null"); + assertNull(throttleSettings.getNamespace(), "Namespace should be null"); + assertNull(throttleSettings.getRegionServer(), "RegionServer should be null"); seenWrite = true; break; default: fail("Unexpected throttle type: " + throttleSettings.getThrottleType()); } } else if (setting instanceof SpaceLimitSettings) { - assertFalse("Should not have multiple space quotas", seenSpace); + assertFalse(seenSpace, "Should not have multiple space quotas"); SpaceLimitSettings spaceLimit = (SpaceLimitSettings) setting; assertEquals(tn, spaceLimit.getTableName()); - assertNull("Username should be null", spaceLimit.getUserName()); - assertNull("Namespace should be null", spaceLimit.getNamespace()); - assertNull("RegionServer should be null", spaceLimit.getRegionServer()); - assertTrue("SpaceLimitSettings should have a SpaceQuota", spaceLimit.getProto().hasQuota()); + assertNull(spaceLimit.getUserName(), "Username should be null"); + assertNull(spaceLimit.getNamespace(), "Namespace should be null"); + assertNull(spaceLimit.getRegionServer(), "RegionServer should be null"); + assertTrue(spaceLimit.getProto().hasQuota(), "SpaceLimitSettings should have a SpaceQuota"); assertEquals(spaceQuota, spaceLimit.getProto().getQuota()); seenSpace = true; } else { fail("Unexpected QuotaSettings implementation: " + setting.getClass()); } } - assertTrue("Should have seen a read quota", seenRead); - assertTrue("Should have seen a write quota", seenWrite); - assertTrue("Should have seen a space quota", seenSpace); + assertTrue(seenRead, "Should have seen a read quota"); + assertTrue(seenWrite, "Should have seen a write quota"); + assertTrue(seenSpace, "Should have seen a space quota"); } - @Test(expected = IllegalArgumentException.class) + @Test public void testNeitherTableNorNamespace() { final SpaceQuota spaceQuota = SpaceQuota.newBuilder().setSoftLimit(1L) .setViolationPolicy(QuotaProtos.SpaceViolationPolicy.DISABLE).build(); - QuotaSettingsFactory.fromSpace(null, null, spaceQuota); + assertThrows(IllegalArgumentException.class, + () -> QuotaSettingsFactory.fromSpace(null, null, spaceQuota)); } - @Test(expected = IllegalArgumentException.class) + @Test public void testBothTableAndNamespace() { final SpaceQuota spaceQuota = SpaceQuota.newBuilder().setSoftLimit(1L) .setViolationPolicy(QuotaProtos.SpaceViolationPolicy.DISABLE).build(); - QuotaSettingsFactory.fromSpace(TableName.valueOf("foo"), "bar", spaceQuota); + assertThrows(IllegalArgumentException.class, + () -> QuotaSettingsFactory.fromSpace(TableName.valueOf("foo"), "bar", spaceQuota)); } @Test @@ -142,16 +140,16 @@ public void testSpaceLimitSettings() { final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_INSERTS; QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tableName, sizeLimit, violationPolicy); - assertNotNull("QuotaSettings should not be null", settings); - assertTrue("Should be an instance of SpaceLimitSettings", - settings instanceof SpaceLimitSettings); + assertNotNull(settings, "QuotaSettings should not be null"); + assertTrue(settings instanceof SpaceLimitSettings, + "Should be an instance of SpaceLimitSettings"); SpaceLimitSettings spaceLimitSettings = (SpaceLimitSettings) settings; SpaceLimitRequest protoRequest = spaceLimitSettings.getProto(); - assertTrue("Request should have a SpaceQuota", protoRequest.hasQuota()); + assertTrue(protoRequest.hasQuota(), "Request should have a SpaceQuota"); SpaceQuota quota = protoRequest.getQuota(); assertEquals(sizeLimit, quota.getSoftLimit()); assertEquals(violationPolicy, ProtobufUtil.toViolationPolicy(quota.getViolationPolicy())); - assertFalse("The remove attribute should be false", quota.getRemove()); + assertFalse(quota.getRemove(), "The remove attribute should be false"); } @Test @@ -159,19 +157,19 @@ public void testSpaceLimitSettingsForDeletes() { final String ns = "ns1"; final TableName tn = TableName.valueOf("tn1"); QuotaSettings nsSettings = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns); - assertNotNull("QuotaSettings should not be null", nsSettings); - assertTrue("Should be an instance of SpaceLimitSettings", - nsSettings instanceof SpaceLimitSettings); + assertNotNull(nsSettings, "QuotaSettings should not be null"); + assertTrue(nsSettings instanceof SpaceLimitSettings, + "Should be an instance of SpaceLimitSettings"); SpaceLimitRequest nsProto = ((SpaceLimitSettings) nsSettings).getProto(); - assertTrue("Request should have a SpaceQuota", nsProto.hasQuota()); - assertTrue("The remove attribute should be true", nsProto.getQuota().getRemove()); + assertTrue(nsProto.hasQuota(), "Request should have a SpaceQuota"); + assertTrue(nsProto.getQuota().getRemove(), "The remove attribute should be true"); QuotaSettings tableSettings = QuotaSettingsFactory.removeTableSpaceLimit(tn); - assertNotNull("QuotaSettings should not be null", tableSettings); - assertTrue("Should be an instance of SpaceLimitSettings", - tableSettings instanceof SpaceLimitSettings); + assertNotNull(tableSettings, "QuotaSettings should not be null"); + assertTrue(tableSettings instanceof SpaceLimitSettings, + "Should be an instance of SpaceLimitSettings"); SpaceLimitRequest tableProto = ((SpaceLimitSettings) tableSettings).getProto(); - assertTrue("Request should have a SpaceQuota", tableProto.hasQuota()); - assertTrue("The remove attribute should be true", tableProto.getQuota().getRemove()); + assertTrue(tableProto.hasQuota(), "Request should have a SpaceQuota"); + assertTrue(tableProto.getQuota().getRemove(), "The remove attribute should be true"); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestRpcThrottlingException.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestRpcThrottlingException.java index 21b01ad4764a..a8f0b4cbda2f 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestRpcThrottlingException.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestRpcThrottlingException.java @@ -17,24 +17,18 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Map; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; -@Category({ SmallTests.class }) +@Tag(SmallTests.TAG) public class TestRpcThrottlingException { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRpcThrottlingException.class); - private static final Map STR_TO_MS_NEW_FORMAT = ImmutableMap. builder().put("0ms", 0L).put("50ms", 50L).put("1sec, 1ms", 1001L) .put("1min, 5sec, 15ms", 65_015L).put("5mins, 2sec, 0ms", 302000L) diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceLimitSettings.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceLimitSettings.java index a3fc235c2e8e..07898295636b 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceLimitSettings.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceLimitSettings.java @@ -17,18 +17,17 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest; @@ -38,43 +37,45 @@ /** * Test class for {@link SpaceLimitSettings}. */ -@Category({ SmallTests.class }) +@Tag(SmallTests.TAG) public class TestSpaceLimitSettings { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSpaceLimitSettings.class); - - @Test(expected = IllegalArgumentException.class) + @Test public void testInvalidTableQuotaSizeLimit() { - new SpaceLimitSettings(TableName.valueOf("foo"), -1, SpaceViolationPolicy.NO_INSERTS); + assertThrows(IllegalArgumentException.class, + () -> new SpaceLimitSettings(TableName.valueOf("foo"), -1, SpaceViolationPolicy.NO_INSERTS)); } - @Test(expected = NullPointerException.class) + @Test public void testNullTableName() { TableName tn = null; - new SpaceLimitSettings(tn, 1, SpaceViolationPolicy.NO_INSERTS); + assertThrows(NullPointerException.class, + () -> new SpaceLimitSettings(tn, 1, SpaceViolationPolicy.NO_INSERTS)); } - @Test(expected = NullPointerException.class) + @Test public void testNullTableViolationPolicy() { - new SpaceLimitSettings(TableName.valueOf("foo"), 1, null); + assertThrows(NullPointerException.class, + () -> new SpaceLimitSettings(TableName.valueOf("foo"), 1, null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void testInvalidNamespaceQuotaSizeLimit() { - new SpaceLimitSettings("foo_ns", -1, SpaceViolationPolicy.NO_INSERTS); + assertThrows(IllegalArgumentException.class, + () -> new SpaceLimitSettings("foo_ns", -1, SpaceViolationPolicy.NO_INSERTS)); } - @Test(expected = NullPointerException.class) + @Test public void testNullNamespace() { String ns = null; - new SpaceLimitSettings(ns, 1, SpaceViolationPolicy.NO_INSERTS); + assertThrows(NullPointerException.class, + () -> new SpaceLimitSettings(ns, 1, SpaceViolationPolicy.NO_INSERTS)); } - @Test(expected = NullPointerException.class) + @Test public void testNullNamespaceViolationPolicy() { - new SpaceLimitSettings("foo_ns", 1, null); + assertThrows(NullPointerException.class, () -> new SpaceLimitSettings("foo_ns", 1, null)); + } @Test @@ -85,13 +86,13 @@ public void testTableQuota() { SpaceLimitSettings settings = new SpaceLimitSettings(tableName, sizeLimit, policy); SetQuotaRequest proto = QuotaSettings.buildSetQuotaRequestProto(settings); - assertFalse("User should be missing", proto.hasUserName()); - assertFalse("Namespace should be missing", proto.hasNamespace()); + assertFalse(proto.hasUserName(), "User should be missing"); + assertFalse(proto.hasNamespace(), "Namespace should be missing"); assertEquals(ProtobufUtil.toProtoTableName(tableName), proto.getTableName()); SpaceLimitRequest spaceLimitReq = proto.getSpaceLimit(); - assertNotNull("SpaceLimitRequest was null", spaceLimitReq); + assertNotNull(spaceLimitReq, "SpaceLimitRequest was null"); SpaceQuota spaceQuota = spaceLimitReq.getQuota(); - assertNotNull("SpaceQuota was null", spaceQuota); + assertNotNull(spaceQuota, "SpaceQuota was null"); assertEquals(sizeLimit, spaceQuota.getSoftLimit()); assertEquals(ProtobufUtil.toProtoViolationPolicy(policy), spaceQuota.getViolationPolicy()); @@ -110,13 +111,13 @@ public void testNamespaceQuota() { SpaceLimitSettings settings = new SpaceLimitSettings(namespace, sizeLimit, policy); SetQuotaRequest proto = QuotaSettings.buildSetQuotaRequestProto(settings); - assertFalse("User should be missing", proto.hasUserName()); - assertFalse("TableName should be missing", proto.hasTableName()); + assertFalse(proto.hasUserName(), "User should be missing"); + assertFalse(proto.hasTableName(), "TableName should be missing"); assertEquals(namespace, proto.getNamespace()); SpaceLimitRequest spaceLimitReq = proto.getSpaceLimit(); - assertNotNull("SpaceLimitRequest was null", spaceLimitReq); + assertNotNull(spaceLimitReq, "SpaceLimitRequest was null"); SpaceQuota spaceQuota = spaceLimitReq.getQuota(); - assertNotNull("SpaceQuota was null", spaceQuota); + assertNotNull(spaceQuota, "SpaceQuota was null"); assertEquals(sizeLimit, spaceQuota.getSoftLimit()); assertEquals(ProtobufUtil.toProtoViolationPolicy(policy), spaceQuota.getViolationPolicy()); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestThrottleSettings.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestThrottleSettings.java index 11e2f737b315..cdcfe24d94cd 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestThrottleSettings.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestThrottleSettings.java @@ -17,30 +17,24 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.ThrottleRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota; -@Category({ SmallTests.class }) +@Tag(SmallTests.TAG) public class TestThrottleSettings { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestThrottleSettings.class); - @Test public void testMerge() throws IOException { TimedQuota tq1 = TimedQuota.newBuilder().setSoftLimit(10) @@ -93,7 +87,7 @@ public void testNoThrottleReturnsOriginal() throws IOException { ThrottleRequest tr2 = ThrottleRequest.newBuilder().setType(QuotaProtos.ThrottleType.REQUEST_NUMBER).build(); - assertTrue("The same object should be returned by merge, but it wasn't", - orig == orig.merge(new ThrottleSettings("joe", null, null, null, tr2))); + assertTrue(orig == orig.merge(new ThrottleSettings("joe", null, null, null, tr2)), + "The same object should be returned by merge, but it wasn't"); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigTestUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigTestUtil.java index 634f4626da52..7a1a9136be50 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigTestUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigTestUtil.java @@ -19,9 +19,9 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashMap; import java.util.Iterator; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java index bdac27067b09..465714635b70 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java @@ -18,15 +18,14 @@ package org.apache.hadoop.hbase.replication; import static org.apache.hadoop.hbase.replication.ReplicationPeerConfigTestUtil.getConfig; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil; @@ -34,21 +33,17 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Maps; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestReplicationPeerConfig { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationPeerConfig.class); - private static final Configuration CONF = HBaseConfiguration.create(); private static final String NAMESPACE_REPLICATE = "replicate"; @@ -268,7 +263,7 @@ public void testBaseReplicationPeerConfig() throws ReplicationException { ReplicationPeerConfig existingReplicationPeerConfig = getConfig(1); // custom config not present - assertEquals(existingReplicationPeerConfig.getConfiguration().get(customPeerConfigKey), null); + assertNull(existingReplicationPeerConfig.getConfiguration().get(customPeerConfigKey)); Configuration conf = new Configuration(CONF); conf.set(ReplicationPeerConfigUtil.HBASE_REPLICATION_PEER_BASE_CONFIG, @@ -306,7 +301,7 @@ public void testBaseReplicationRemovePeerConfig() throws ReplicationException { ReplicationPeerConfig existingReplicationPeerConfig = getConfig(1); // custom config not present - assertEquals(existingReplicationPeerConfig.getConfiguration().get(customPeerConfigKey), null); + assertNull(existingReplicationPeerConfig.getConfiguration().get(customPeerConfigKey)); Configuration conf = new Configuration(CONF); conf.set(ReplicationPeerConfigUtil.HBASE_REPLICATION_PEER_BASE_CONFIG, @@ -336,7 +331,7 @@ public void testBaseReplicationRemovePeerConfigWithNoExistingConfig() ReplicationPeerConfig existingReplicationPeerConfig = getConfig(1); // custom config not present - assertEquals(existingReplicationPeerConfig.getConfiguration().get(customPeerConfigKey), null); + assertNull(existingReplicationPeerConfig.getConfiguration().get(customPeerConfigKey)); Configuration conf = new Configuration(CONF); conf.set(ReplicationPeerConfigUtil.HBASE_REPLICATION_PEER_BASE_CONFIG, customPeerConfigKey.concat("=").concat("")); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java index 6f7be8315853..51f47dd5df14 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java @@ -17,15 +17,15 @@ */ package org.apache.hadoop.hbase.security; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.security.Key; import java.security.KeyException; import javax.crypto.spec.SecretKeySpec; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; @@ -33,20 +33,16 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, SmallTests.class }) +@Tag(ClientTests.TAG) +@Tag(SmallTests.TAG) public class TestEncryptionUtil { private static final String INVALID_HASH_ALG = "this-hash-algorithm-not-exists hopefully... :)"; private static final String DEFAULT_HASH_ALGORITHM = "use-default"; - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestEncryptionUtil.class); - // There does not seem to be a ready way to test either getKeyFromBytesOrMasterKey // or createEncryptionContext, and the existing code under MobUtils appeared to be // untested. Not ideal! @@ -71,9 +67,9 @@ public void testKeyWrappingUsingHashAlgSHA384() throws Exception { testKeyWrapping("SHA-384"); } - @Test(expected = RuntimeException.class) + @Test public void testKeyWrappingWithInvalidHashAlg() throws Exception { - testKeyWrapping(INVALID_HASH_ALG); + assertThrows(RuntimeException.class, () -> testKeyWrapping(INVALID_HASH_ALG)); } @Test @@ -96,12 +92,12 @@ public void testWALKeyWrappingUsingHashAlgSHA384() throws Exception { testWALKeyWrapping("SHA-384"); } - @Test(expected = RuntimeException.class) + @Test public void testWALKeyWrappingWithInvalidHashAlg() throws Exception { - testWALKeyWrapping(INVALID_HASH_ALG); + assertThrows(RuntimeException.class, () -> testWALKeyWrapping(INVALID_HASH_ALG)); } - @Test(expected = KeyException.class) + @Test public void testWALKeyWrappingWithIncorrectKey() throws Exception { // set up the key provider for testing to resolve a key for our test subject Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this @@ -118,14 +114,15 @@ public void testWALKeyWrappingWithIncorrectKey() throws Exception { assertNotNull(wrappedKeyBytes); // unwrap with an incorrect key - EncryptionUtil.unwrapWALKey(conf, "other", wrappedKeyBytes); + assertThrows(KeyException.class, + () -> EncryptionUtil.unwrapWALKey(conf, "other", wrappedKeyBytes)); } - @Test(expected = KeyException.class) + @Test public void testHashAlgorithmMismatchWhenFailExpected() throws Exception { Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this conf.setBoolean(Encryption.CRYPTO_KEY_FAIL_ON_ALGORITHM_MISMATCH_CONF_KEY, true); - testKeyWrappingWithMismatchingAlgorithms(conf); + assertThrows(KeyException.class, () -> testKeyWrappingWithMismatchingAlgorithms(conf)); } @Test @@ -165,8 +162,8 @@ private void testKeyWrapping(String hashAlgorithm) throws Exception { // only secretkeyspec supported for now assertTrue(unwrappedKey instanceof SecretKeySpec); // did we get back what we wrapped? - assertTrue("Unwrapped key bytes do not match original", - Bytes.equals(keyBytes, unwrappedKey.getEncoded())); + assertTrue(Bytes.equals(keyBytes, unwrappedKey.getEncoded()), + "Unwrapped key bytes do not match original"); // unwrap with an incorrect key try { @@ -201,8 +198,8 @@ private void testWALKeyWrapping(String hashAlgorithm) throws Exception { // only secretkeyspec supported for now assertTrue(unwrappedKey instanceof SecretKeySpec); // did we get back what we wrapped? - assertTrue("Unwrapped key bytes do not match original", - Bytes.equals(keyBytes, unwrappedKey.getEncoded())); + assertTrue(Bytes.equals(keyBytes, unwrappedKey.getEncoded()), + "Unwrapped key bytes do not match original"); } private void testKeyWrappingWithMismatchingAlgorithms(Configuration conf) throws Exception { @@ -230,8 +227,8 @@ private void testKeyWrappingWithMismatchingAlgorithms(Configuration conf) throws assertNotNull(unwrappedKey); // did we get back what we wrapped? - assertTrue("Unwrapped key bytes do not match original", - Bytes.equals(keyBytes, unwrappedKey.getEncoded())); + assertTrue(Bytes.equals(keyBytes, unwrappedKey.getEncoded()), + "Unwrapped key bytes do not match original"); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestHBaseSaslRpcClient.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestHBaseSaslRpcClient.java index 6b1e7c338329..1d0144d6fd49 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestHBaseSaslRpcClient.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestHBaseSaslRpcClient.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.security; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -39,7 +39,6 @@ import javax.security.sasl.Sasl; import javax.security.sasl.SaslClient; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.security.provider.DigestSaslClientAuthenticationProvider; import org.apache.hadoop.hbase.security.provider.DigestSaslClientAuthenticationProvider.DigestSaslClientCallbackHandler; @@ -52,22 +51,18 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.base.Strings; -@Category({ SecurityTests.class, SmallTests.class }) +@Tag(SecurityTests.TAG) +@Tag(SmallTests.TAG) public class TestHBaseSaslRpcClient { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHBaseSaslRpcClient.class); - static { System.setProperty("java.security.krb5.realm", "DOMAIN.COM"); System.setProperty("java.security.krb5.kdc", "DOMAIN.COM"); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestSaslUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestSaslUtil.java index 9e0b28b131b2..e4e1294980ad 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestSaslUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestSaslUtil.java @@ -17,31 +17,21 @@ */ package org.apache.hadoop.hbase.security; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; import java.util.Map; import javax.security.sasl.Sasl; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ SecurityTests.class, SmallTests.class }) +@Tag(SecurityTests.TAG) +@Tag(SmallTests.TAG) public class TestSaslUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSaslUtil.class); - - @Rule - public ExpectedException exception = ExpectedException.none(); - @Test public void testInitSaslProperties() { Map props; @@ -55,11 +45,11 @@ public void testInitSaslProperties() { props = SaslUtil.initSaslProperties("integrity,authentication,privacy"); assertEquals("auth-int,auth,auth-conf", props.get(Sasl.QOP)); - exception.expect(IllegalArgumentException.class); - props = SaslUtil.initSaslProperties("xyz"); - assertEquals("auth", props.get(Sasl.QOP)); + assertThrows(IllegalArgumentException.class, () -> { + Map invalidProps = SaslUtil.initSaslProperties("xyz"); + assertEquals("auth", invalidProps.get(Sasl.QOP)); + }); - exception.expect(IllegalArgumentException.class); props = SaslUtil.initSaslProperties(""); assertEquals("auth", props.get(Sasl.QOP)); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/provider/TestDefaultProviderSelector.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/provider/TestDefaultProviderSelector.java index 52ebebc372fa..dc17d9a6bad5 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/provider/TestDefaultProviderSelector.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/provider/TestDefaultProviderSelector.java @@ -17,55 +17,53 @@ */ package org.apache.hadoop.hbase.security.provider; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ SmallTests.class }) +@Tag(SmallTests.TAG) public class TestDefaultProviderSelector { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestDefaultProviderSelector.class); - BuiltInProviderSelector selector; - @Before + @BeforeEach public void setup() { selector = new BuiltInProviderSelector(); } - @Test(expected = IllegalStateException.class) + @Test public void testExceptionOnMissingProviders() { - selector.configure(new Configuration(false), Collections.emptySet()); + assertThrows(IllegalStateException.class, + () -> selector.configure(new Configuration(false), Collections.emptySet())); } - @Test(expected = NullPointerException.class) + @Test public void testNullConfiguration() { - selector.configure(null, Collections.emptySet()); + assertThrows(NullPointerException.class, + () -> selector.configure(null, Collections.emptySet())); } - @Test(expected = NullPointerException.class) + @Test public void testNullProviderMap() { - selector.configure(new Configuration(false), null); + assertThrows(NullPointerException.class, () -> selector.configure(new Configuration(), null)); } - @Test(expected = IllegalStateException.class) + @Test public void testDuplicateProviders() { Set providers = new HashSet<>(); providers.add(new SimpleSaslClientAuthenticationProvider()); providers.add(new SimpleSaslClientAuthenticationProvider()); - selector.configure(new Configuration(false), providers); + assertThrows(IllegalStateException.class, + () -> selector.configure(new Configuration(false), providers)); } @Test @@ -76,8 +74,8 @@ public void testExpectedProviders() { selector.configure(new Configuration(false), providers); - assertNotNull("Simple provider was null", selector.simpleAuth); - assertNotNull("Kerberos provider was null", selector.krbAuth); - assertNotNull("Digest provider was null", selector.digestAuth); + assertNotNull(selector.simpleAuth, "Simple provider was null"); + assertNotNull(selector.krbAuth, "Kerberos provider was null"); + assertNotNull(selector.digestAuth, "Digest provider was null"); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/provider/TestSaslClientAuthenticationProviders.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/provider/TestSaslClientAuthenticationProviders.java index 029c880600b4..9b4fdef06133 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/provider/TestSaslClientAuthenticationProviders.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/provider/TestSaslClientAuthenticationProviders.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.security.provider; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertSame; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; import java.net.InetAddress; @@ -27,7 +27,6 @@ import java.util.Map; import javax.security.sasl.SaslClient; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.security.SecurityInfo; import org.apache.hadoop.hbase.security.User; @@ -36,19 +35,15 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.UserInformation; -@Category({ SmallTests.class, SecurityTests.class }) +@Tag(SmallTests.TAG) +@Tag(SecurityTests.TAG) public class TestSaslClientAuthenticationProviders { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSaslClientAuthenticationProviders.class); - @Test public void testCannotAddTheSameProviderTwice() { HashMap registeredProviders = new HashMap<>(); @@ -58,38 +53,19 @@ public void testCannotAddTheSameProviderTwice() { SaslClientAuthenticationProviders.addProviderIfNotExists(p1, registeredProviders); assertEquals(1, registeredProviders.size()); - try { - SaslClientAuthenticationProviders.addProviderIfNotExists(p2, registeredProviders); - } catch (RuntimeException e) { - } + assertThrows(RuntimeException.class, + () -> SaslClientAuthenticationProviders.addProviderIfNotExists(p2, registeredProviders)); - assertSame("Expected the original provider to be present", p1, - registeredProviders.entrySet().iterator().next().getValue()); + assertSame(p1, registeredProviders.entrySet().iterator().next().getValue(), + "Expected the original provider to be present"); } @Test - public void testInstanceIsCached() { - Configuration conf = HBaseConfiguration.create(); - SaslClientAuthenticationProviders providers1 = - SaslClientAuthenticationProviders.getInstance(conf); - SaslClientAuthenticationProviders providers2 = - SaslClientAuthenticationProviders.getInstance(conf); - assertSame(providers1, providers2); - - SaslClientAuthenticationProviders.reset(); - - SaslClientAuthenticationProviders providers3 = - SaslClientAuthenticationProviders.getInstance(conf); - assertNotSame(providers1, providers3); - assertEquals(providers1.getNumRegisteredProviders(), providers3.getNumRegisteredProviders()); - } - - @Test(expected = RuntimeException.class) public void testDifferentConflictingImplementationsFail() { Configuration conf = HBaseConfiguration.create(); conf.setStrings(SaslClientAuthenticationProviders.EXTRA_PROVIDERS_KEY, ConflictingProvider1.class.getName(), ConflictingProvider2.class.getName()); - SaslClientAuthenticationProviders.getInstance(conf); + assertThrows(RuntimeException.class, () -> new SaslClientAuthenticationProviders(conf)); } static class ConflictingProvider1 implements SaslClientAuthenticationProvider { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/token/TestClientTokenUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/token/TestClientTokenUtil.java index a2f26e073e76..f9a4b6364f03 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/token/TestClientTokenUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/token/TestClientTokenUtil.java @@ -17,43 +17,37 @@ */ package org.apache.hadoop.hbase.security.token; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.lang.reflect.Field; import java.net.URL; import java.net.URLClassLoader; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestClientTokenUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestClientTokenUtil.class); - private URLClassLoader cl; - @Before + @BeforeEach public void setUp() { URL urlPU = ProtobufUtil.class.getProtectionDomain().getCodeSource().getLocation(); URL urlCTU = ClientTokenUtil.class.getProtectionDomain().getCodeSource().getLocation(); cl = new URLClassLoader(new URL[] { urlPU, urlCTU }, getClass().getClassLoader()); } - @After + @AfterEach public void tearDown() throws IOException { Closeables.close(cl, true); } @@ -86,6 +80,6 @@ public void testObtainToken() throws Exception { Boolean loaded = (Boolean) cl.loadClass(ProtobufUtil.class.getCanonicalName()) .getDeclaredMethod("isClassLoaderLoaded").invoke(null); - assertFalse("Should not have loaded DynamicClassLoader", loaded); + assertFalse(loaded, "Should not have loaded DynamicClassLoader"); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java index a68fc58e6ffb..8dd124b3cec4 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.shaded.protobuf; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.lang.reflect.Method; @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.ExtendedCellBuilder; import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -47,9 +46,7 @@ import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.protobuf.Any; @@ -72,11 +69,9 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos; -@Category(SmallTests.class) +@org.junit.jupiter.api.Tag(SmallTests.TAG) public class TestProtobufUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProtobufUtil.class); + private static final String TAG_STR = "tag-1"; private static final byte TAG_TYPE = (byte) 10; private static final HBaseProtos.ServerName SERVER_NAME = diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/BuilderStyleTest.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/BuilderStyleTest.java index cae296b4d0e3..05c0e690eeff 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/BuilderStyleTest.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/BuilderStyleTest.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -84,7 +84,7 @@ public static void assertClassesAreBuilderStyle(Class... classes) { String errorMsg = "All setXXX()|addXX() methods in " + clazz.getSimpleName() + " should return a " + clazz.getSimpleName() + " object in builder style. " + "Offending method:" + e.getValue().iterator().next().getName(); - assertTrue(errorMsg, found); + assertTrue(found, errorMsg); } } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/PoolMapTestBase.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/PoolMapTestBase.java index fbf92f641392..dea2c7077b20 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/PoolMapTestBase.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/PoolMapTestBase.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.util; import org.apache.hadoop.hbase.util.PoolMap.PoolType; -import org.junit.Before; +import org.junit.jupiter.api.BeforeEach; public abstract class PoolMapTestBase { @@ -27,7 +27,7 @@ public abstract class PoolMapTestBase { protected static final int KEY_COUNT = 5; protected static final int POOL_SIZE = 3; - @Before + @BeforeEach public void setUp() throws Exception { this.poolMap = new PoolMap<>(getPoolType(), POOL_SIZE); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestRotateFile.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestRotateFile.java index 5f36d468350d..8020cd75c09a 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestRotateFile.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestRotateFile.java @@ -19,37 +19,31 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.startsWith; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestRotateFile { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRotateFile.class); - private static HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility(); private static FileSystem FS; @@ -58,26 +52,24 @@ public class TestRotateFile { private RotateFile rotateFile; - @Rule - public final TestName name = new TestName(); - - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws IOException { FS = FileSystem.get(UTIL.getConfiguration()); } - @AfterClass + @AfterAll public static void tearDownAfterClass() { UTIL.cleanupTestDir(); } - @Before - public void setUp() throws IOException { - dir = UTIL.getDataTestDir(name.getMethodName()); + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException { + String methodName = testInfo.getTestMethod().get().getName(); + dir = UTIL.getDataTestDir(methodName); if (!FS.mkdirs(dir)) { throw new IOException("Can not create dir " + dir); } - rotateFile = new RotateFile(FS, dir, name.getMethodName(), 1024); + rotateFile = new RotateFile(FS, dir, methodName, 1024); assertNull(rotateFile.read()); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestRoundRobinPoolMap.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestRoundRobinPoolMap.java index 04ba2570eac8..87e861dbea57 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestRoundRobinPoolMap.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestRoundRobinPoolMap.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.io.IOException; import java.util.ArrayList; @@ -29,21 +29,16 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.PoolMap.PoolType; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestRoundRobinPoolMap extends PoolMapTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRoundRobinPoolMap.class); - @Override protected PoolType getPoolType() { return PoolType.RoundRobin; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestThreadLocalPoolMap.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestThreadLocalPoolMap.java index d83944ef7f3a..2b5cb36072a8 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestThreadLocalPoolMap.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/util/TestThreadLocalPoolMap.java @@ -17,28 +17,23 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.PoolMap.PoolType; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestThreadLocalPoolMap extends PoolMapTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestThreadLocalPoolMap.class); - @Override protected PoolType getPoolType() { return PoolType.ThreadLocal; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZNodePaths.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZNodePaths.java index c6d61ca44574..7ea8c2ebc29d 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZNodePaths.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZNodePaths.java @@ -17,24 +17,19 @@ */ package org.apache.hadoop.hbase.zookeeper; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.testclassification.ZKTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ZKTests.class, SmallTests.class }) +@Tag(ZKTests.TAG) +@Tag(SmallTests.TAG) public class TestZNodePaths { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestZNodePaths.class); - @Test public void testIsClientReadable() { ZNodePaths znodePaths = new ZNodePaths(HBaseConfiguration.create()); diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 6f3cdf66eede..7a6174553df2 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -110,8 +110,28 @@ commons-crypto - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility test @@ -130,6 +150,11 @@ mockito-core test + + org.mockito + mockito-inline + test + org.slf4j jcl-over-slf4j diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 14d7073d5da5..fbe563d60d03 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1089,7 +1089,8 @@ public enum OperationStatusCode { public static final boolean HFILE_PREAD_ALL_BYTES_ENABLED_DEFAULT = false; /* - * Minimum percentage of free heap necessary for a successful cluster startup. + * Default minimum fraction (20%) of free heap required for RegionServer startup, used only when + * 'hbase.regionserver.free.heap.min.memory.size' is not explicitly set. */ public static final float HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD = 0.2f; @@ -1269,6 +1270,8 @@ public enum OperationStatusCode { public static final int ADMIN_QOS = 100; public static final int HIGH_QOS = 200; public static final int SYSTEMTABLE_QOS = HIGH_QOS; + // QOS for internal meta read requests + public static final int INTERNAL_READ_QOS = 250; /** * @deprecated the name "META_QOS" is a bit ambiguous, actually only meta region transition can * use this priority, and you should not use this directly. Will be removed in 3.0.0. @@ -1619,6 +1622,18 @@ public enum OperationStatusCode { // User defined Default TTL config key public static final String DEFAULT_SNAPSHOT_TTL_CONFIG_KEY = "hbase.master.snapshot.ttl"; + // Soft drop for destructive table actions configuration + public static final String SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_ENABLED_KEY = + "hbase.snapshot.before.destructive.action.enabled"; + public static final boolean DEFAULT_SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_ENABLED = false; + + public static final String SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_TTL_KEY = + "hbase.snapshot.before.destructive.action.ttl"; + public static final long DEFAULT_SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_TTL = 86400; // 1 day + + // Table-level attribute name for recovery snapshot TTL override + public static final String TABLE_RECOVERY_SNAPSHOT_TTL_KEY = "RECOVERY_SNAPSHOT_TTL"; + // Regions Recovery based on high storeFileRefCount threshold value public static final String STORE_FILE_REF_COUNT_THRESHOLD = "hbase.regions.recovery.store.file.ref.count"; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java index eb9a7f3eccc9..b0df4920e4ed 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java @@ -36,4 +36,6 @@ public final class TagType { // String based tag type used in replication public static final byte STRING_VIS_TAG_TYPE = (byte) 7; public static final byte TTL_TAG_TYPE = (byte) 8; + // tag with the custom cell tiering value for the row + public static final byte CELL_VALUE_TIERING_TAG_TYPE = (byte) 9; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/FileChangeWatcher.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/FileChangeWatcher.java index 3ac4076766b8..fbcdbf41f816 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/FileChangeWatcher.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/FileChangeWatcher.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.io; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -173,8 +174,10 @@ public void stop() { } } - String getWatcherThreadName() { - return watcherThread.getName(); + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + Thread getWatcherThread() { + return watcherThread; } private static void handleException(Thread thread, Throwable e) { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java index f938fdaab35b..09f8c23a8e51 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java @@ -20,12 +20,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.io.util.Dictionary; import org.apache.hadoop.hbase.io.util.StreamUtils; +import org.apache.hadoop.hbase.io.util.UndoableLRUDictionary; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -38,13 +37,10 @@ */ @InterfaceAudience.Private public class TagCompressionContext { - private final Dictionary tagDict; + private final UndoableLRUDictionary tagDict; - public TagCompressionContext(Class dictType, int dictCapacity) - throws SecurityException, NoSuchMethodException, InstantiationException, IllegalAccessException, - InvocationTargetException { - Constructor dictConstructor = dictType.getConstructor(); - tagDict = dictConstructor.newInstance(); + public TagCompressionContext(int dictCapacity) { + tagDict = new UndoableLRUDictionary(); tagDict.init(dictCapacity); } @@ -52,6 +48,18 @@ public void clear() { tagDict.clear(); } + public void checkpoint() { + tagDict.checkpoint(); + } + + public void commit() { + tagDict.commit(); + } + + public void rollback() { + tagDict.rollback(); + } + /** * Compress tags one by one and writes to the OutputStream. * @param out Stream to which the compressed tags to be written diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/GzipByteBuffDecompressor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/GzipByteBuffDecompressor.java new file mode 100644 index 000000000000..92ff19a183d0 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/GzipByteBuffDecompressor.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.compress; + +import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.hbase.nio.SingleByteBuff; +import org.apache.hadoop.io.compress.zlib.ZlibDecompressor; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Glue for ByteBuffDecompressor on top of Hadoop's native + * {@link ZlibDecompressor.ZlibDirectDecompressor}. Only direct-to-direct decompression is + * supported, which is zero-copy; callers with on-heap buffers fall back to the stream path. + */ +@InterfaceAudience.Private +public class GzipByteBuffDecompressor implements ByteBuffDecompressor { + + private static final int GZIP_HEADER_LENGTH = 10; + private static final int GZIP_TRAILER_LENGTH = 8; + + @Nullable + private final ZlibDecompressor.ZlibDirectDecompressor decompressor; + + private boolean allowByteBuffDecompression; + + GzipByteBuffDecompressor(boolean nativeZlibLoaded) { + decompressor = nativeZlibLoaded + ? new ZlibDecompressor.ZlibDirectDecompressor(ZlibDecompressor.CompressionHeader.AUTODETECT_GZIP_ZLIB, + 0) + : null; + allowByteBuffDecompression = true; + } + + @Override + public boolean canDecompress(ByteBuff output, ByteBuff input) { + if (!allowByteBuffDecompression) { + return false; + } + if (!(output instanceof SingleByteBuff) || !(input instanceof SingleByteBuff)) { + return false; + } + // Only direct-to-direct decompression is supported. + return input.nioByteBuffers()[0].isDirect() && output.nioByteBuffers()[0].isDirect() + && decompressor != null; + } + + @Override + public int decompress(ByteBuff output, ByteBuff input, int inputLen) throws IOException { + if (!(output instanceof SingleByteBuff) || !(input instanceof SingleByteBuff)) { + throw new IllegalStateException( + "At least one buffer is not a SingleByteBuff, this is not supported"); + } + if (inputLen < GZIP_HEADER_LENGTH + GZIP_TRAILER_LENGTH) { + throw new IOException("Input of length " + inputLen + " is too short to be a gzip member"); + } + + ByteBuffer nioInput = input.nioByteBuffers()[0]; + ByteBuffer nioOutput = output.nioByteBuffers()[0]; + if (!nioInput.isDirect() || !nioOutput.isDirect() || decompressor == null) { + throw new IllegalStateException( + "GzipByteBuffDecompressor only supports direct-to-direct decompression with native zlib " + + "loaded, this should never happen since canDecompress() would have returned false"); + } + return decompressOffHeap(nioInput, nioOutput, inputLen); + } + + private int decompressOffHeap(ByteBuffer nioInput, ByteBuffer nioOutput, int inputLen) + throws IOException { + int inputStart = nioInput.position(); + int outputStart = nioOutput.position(); + + ByteBuffer gzipMember = nioInput.duplicate(); + gzipMember.limit(inputStart + inputLen); + + decompressor.reset(); + try { + decompressor.decompress(gzipMember, nioOutput); + } catch (IOException e) { + throw new IOException("Invalid gzip stream: " + e.getMessage(), e); + } + if (!decompressor.finished()) { + if (!nioOutput.hasRemaining()) { + throw new IOException("Output buffer is too small for the decompressed gzip stream"); + } + throw new IOException("Unexpected end of gzip stream"); + } + if (gzipMember.hasRemaining()) { + throw new IOException("Unexpected trailing bytes after decompressing gzip stream"); + } + + nioInput.position(inputStart + inputLen); + + return nioOutput.position() - outputStart; + } + + @Override + public void reinit(@Nullable Compression.HFileDecompressionContext newHFileDecompressionContext) { + if (newHFileDecompressionContext == null) { + return; + } + if (!(newHFileDecompressionContext instanceof GzipHFileDecompressionContext)) { + throw new IllegalArgumentException( + "GzipByteBuffDecompressor#reinit() was given an HFileDecompressionContext that was not " + + "a GzipHFileDecompressionContext, this should never happen"); + } + GzipHFileDecompressionContext gzipContext = + (GzipHFileDecompressionContext) newHFileDecompressionContext; + allowByteBuffDecompression = gzipContext.isAllowByteBuffDecompression(); + } + + @Override + public void close() { + if (decompressor != null) { + decompressor.end(); + } + } + +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/GzipHFileDecompressionContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/GzipHFileDecompressionContext.java new file mode 100644 index 000000000000..69bdc7ed10ba --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/GzipHFileDecompressionContext.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.compress; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Holds HFile-level settings used by GzipByteBuffDecompressor. It's expensive to pull these from a + * Configuration object every time we decompress a block, so pull them upon opening an HFile, and + * reuse them in every block that gets decompressed. + */ +@InterfaceAudience.Private +public final class GzipHFileDecompressionContext extends Compression.HFileDecompressionContext { + + public static final long FIXED_OVERHEAD = + ClassSize.estimateBase(GzipHFileDecompressionContext.class, false); + + public static final String ALLOW_BYTE_BUFF_DECOMPRESSION_KEY = + "hbase.io.compress.gz.allowByteBuffDecompression"; + + private final boolean allowByteBuffDecompression; + + private GzipHFileDecompressionContext(boolean allowByteBuffDecompression) { + this.allowByteBuffDecompression = allowByteBuffDecompression; + } + + public boolean isAllowByteBuffDecompression() { + return allowByteBuffDecompression; + } + + public static GzipHFileDecompressionContext fromConfiguration(Configuration conf) { + return new GzipHFileDecompressionContext( + conf.getBoolean(ALLOW_BYTE_BUFF_DECOMPRESSION_KEY, true)); + } + + @Override + public void close() throws IOException { + } + + @Override + public long heapSize() { + return FIXED_OVERHEAD; + } + + @Override + public String toString() { + return "GzipHFileDecompressionContext{allowByteBuffDecompression=" + allowByteBuffDecompression + + '}'; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/ReusableStreamGzipCodec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/ReusableStreamGzipCodec.java index 0b3b3afbfc58..08276d7f0732 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/ReusableStreamGzipCodec.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/ReusableStreamGzipCodec.java @@ -22,6 +22,7 @@ import java.io.OutputStream; import java.util.Arrays; import java.util.zip.GZIPOutputStream; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.JVM; import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.CompressorStream; @@ -35,7 +36,7 @@ * Fixes an inefficiency in Hadoop's Gzip codec, allowing to reuse compression streams. */ @InterfaceAudience.Private -public class ReusableStreamGzipCodec extends GzipCodec { +public class ReusableStreamGzipCodec extends GzipCodec implements ByteBuffDecompressionCodec { private static final Logger LOG = LoggerFactory.getLogger(Compression.class); @@ -185,4 +186,20 @@ public CompressionOutputStream createOutputStream(OutputStream out) throws IOExc return new ReusableGzipOutputStream(out); } + @Override + public ByteBuffDecompressor createByteBuffDecompressor() { + return new GzipByteBuffDecompressor(ZlibFactory.isNativeZlibLoaded(getConf())); + } + + @Override + public Class getByteBuffDecompressorType() { + return GzipByteBuffDecompressor.class; + } + + @Override + public Compression.HFileDecompressionContext + getDecompressionContextFromConfiguration(Configuration conf) { + return GzipHFileDecompressionContext.fromConfiguration(conf); + } + } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index 0f15151fe88b..2b11f24578e0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.io.TagCompressionContext; -import org.apache.hadoop.hbase.io.util.LRUDictionary; import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; @@ -72,8 +71,7 @@ public ByteBuffer decodeKeyValues(DataInputStream source, decodingCtx.getTagCompressionContext().clear(); } else { try { - TagCompressionContext tagCompressionContext = - new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext tagCompressionContext = new TagCompressionContext(Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); @@ -815,7 +813,7 @@ public BufferedEncodedSeeker(HFileBlockDecodingContext decodingCtx) { super(decodingCtx); if (decodingCtx.getHFileContext().isCompressTags()) { try { - tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + tagCompressionContext = new TagCompressionContext(Byte.MAX_VALUE); } catch (Exception e) { throw new RuntimeException("Failed to initialize TagCompressionContext", e); } @@ -1232,8 +1230,7 @@ public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOut encodingCtx.getTagCompressionContext().clear(); } else { try { - TagCompressionContext tagCompressionContext = - new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext tagCompressionContext = new TagCompressionContext(Byte.MAX_VALUE); encodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/LRUDictionary.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/LRUDictionary.java index 4089863d4387..41eb7297294a 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/LRUDictionary.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/LRUDictionary.java @@ -62,7 +62,7 @@ public short addEntry(byte[] data, int offset, int length) { return addEntryInternal(data, offset, length, true); } - private short addEntryInternal(byte[] data, int offset, int length, boolean copy) { + short addEntryInternal(byte[] data, int offset, int length, boolean copy) { if (length <= 0) return NOT_IN_DICTIONARY; return backingStore.put(data, offset, length, copy); } @@ -77,22 +77,22 @@ public void clear() { * thread safe. Don't use in multi-threaded applications. */ static class BidirectionalLRUMap { - private int currSize = 0; + int currSize = 0; // Head and tail of the LRU list. - private Node head; - private Node tail; + Node head; + Node tail; - private HashMap nodeToIndex = new HashMap<>(); - private Node[] indexToNode; - private int initSize = 0; + HashMap nodeToIndex = new HashMap<>(); + Node[] indexToNode; + int initSize = 0; public BidirectionalLRUMap(int initialSize) { initSize = initialSize; indexToNode = new Node[initialSize]; } - private short put(byte[] array, int offset, int length, boolean copy) { + short put(byte[] array, int offset, int length, boolean copy) { if (copy) { // We copy the bytes we want, otherwise we might be holding references to // massive arrays in our dictionary (or those arrays might change) @@ -104,7 +104,7 @@ private short put(byte[] array, int offset, int length, boolean copy) { } } - private short putInternal(byte[] stored) { + short putInternal(byte[] stored) { if (currSize < initSize) { // There is space to add without evicting. if (indexToNode[currSize] == null) { @@ -125,7 +125,7 @@ private short putInternal(byte[] stored) { } } - private short findIdx(byte[] array, int offset, int length) { + short findIdx(byte[] array, int offset, int length) { Short s; final Node comparisonNode = new ByteArrayBackedNode(); comparisonNode.setContents(array, offset, length); @@ -137,7 +137,7 @@ private short findIdx(byte[] array, int offset, int length) { } } - private short findIdx(ByteBuffer buf, int offset, int length) { + short findIdx(ByteBuffer buf, int offset, int length) { Short s; final ByteBufferBackedNode comparisonNode = new ByteBufferBackedNode(); comparisonNode.setContents(buf, offset, length); @@ -149,13 +149,13 @@ private short findIdx(ByteBuffer buf, int offset, int length) { } } - private byte[] get(short idx) { + byte[] get(short idx) { Preconditions.checkElementIndex(idx, currSize); moveToHead(indexToNode[idx]); return indexToNode[idx].getContents(); } - private void moveToHead(Node n) { + void moveToHead(Node n) { if (head == n) { // no-op -- it's already the head. return; @@ -176,7 +176,7 @@ private void moveToHead(Node n) { setHead(n); } - private void setHead(Node n) { + void setHead(Node n) { // assume it's already unlinked from the list at this point. n.prev = null; n.next = head; @@ -193,7 +193,7 @@ private void setHead(Node n) { } } - private void clear() { + void clear() { for (int i = 0; i < currSize; i++) { indexToNode[i].next = null; indexToNode[i].prev = null; @@ -205,7 +205,7 @@ private void clear() { head = null; } - private static abstract class Node { + static abstract class Node { int offset; int length; Node next; // link towards the tail @@ -219,7 +219,7 @@ private static abstract class Node { } // The actual contents of the LRUDictionary are of ByteArrayBackedNode type - private static class ByteArrayBackedNode extends Node { + static class ByteArrayBackedNode extends Node { private byte[] container; @Override @@ -259,7 +259,7 @@ public boolean equals(Object other) { // Currently only used for finding the index and hence this node acts // as a temporary holder to look up in the indexToNode map // which is formed by ByteArrayBackedNode - private static class ByteBufferBackedNode extends Node { + static class ByteBufferBackedNode extends Node { private ByteBuffer container; public ByteBufferBackedNode() { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/UndoableLRUDictionary.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/UndoableLRUDictionary.java new file mode 100644 index 000000000000..bc1428ded80a --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/UndoableLRUDictionary.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.util; + +import java.util.IdentityHashMap; +import java.util.Map; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * An LRUDictionary that supports checkpoint and rollback. Used for tag dictionary compression in + * WAL decoding, where dictionary updates within a single cell must be rolled back if the cell read + * fails (e.g., due to EOF on a WAL being tailed). + *

+ * On {@link #checkpoint()}, saves the current LRU state. Operations proceed normally against the + * real dictionary. On {@link #rollback()}, restores the dictionary to the checkpointed state. On + * {@link #commit()}, discards the saved state. + */ +@InterfaceAudience.Private +public class UndoableLRUDictionary extends LRUDictionary { + + private boolean tracking = false; + private int savedCurrSize; + private BidirectionalLRUMap.Node savedHead; + private BidirectionalLRUMap.Node savedTail; + private final Map snapshots = new IdentityHashMap<>(); + + private static class NodeSnapshot { + final BidirectionalLRUMap.Node savedPrev; + final BidirectionalLRUMap.Node savedNext; + final byte[] savedContents; + final int savedOffset; + final int savedLength; + + NodeSnapshot(BidirectionalLRUMap.Node node) { + this.savedPrev = node.prev; + this.savedNext = node.next; + this.savedContents = node.getContents(); + this.savedOffset = node.offset; + this.savedLength = node.length; + } + } + + public void checkpoint() { + tracking = true; + savedCurrSize = backingStore.currSize; + savedHead = backingStore.head; + savedTail = backingStore.tail; + snapshots.clear(); + } + + public void commit() { + snapshots.clear(); + tracking = false; + } + + public void rollback() { + if (!tracking) { + return; + } + for (Map.Entry entry : snapshots.entrySet()) { + BidirectionalLRUMap.Node node = entry.getKey(); + NodeSnapshot snap = entry.getValue(); + node.prev = snap.savedPrev; + node.next = snap.savedNext; + if (snap.savedContents != null) { + node.setContents(snap.savedContents, snap.savedOffset, snap.savedLength); + } + } + backingStore.head = savedHead; + backingStore.tail = savedTail; + backingStore.currSize = savedCurrSize; + backingStore.nodeToIndex.clear(); + for (short i = 0; i < savedCurrSize; i++) { + backingStore.nodeToIndex.put(backingStore.indexToNode[i], i); + } + snapshots.clear(); + tracking = false; + } + + private void saveIfNeeded(BidirectionalLRUMap.Node node) { + if (tracking && node != null && !snapshots.containsKey(node)) { + snapshots.put(node, new NodeSnapshot(node)); + } + } + + @Override + public byte[] getEntry(short idx) { + if (tracking) { + BidirectionalLRUMap.Node node = backingStore.indexToNode[idx]; + saveIfNeeded(node); + if (node.prev != null) { + saveIfNeeded(node.prev); + } + if (node.next != null) { + saveIfNeeded(node.next); + } + saveIfNeeded(backingStore.head); + } + return backingStore.get(idx); + } + + @Override + public short addEntry(byte[] data, int offset, int length) { + if (tracking) { + if (backingStore.currSize < backingStore.initSize) { + BidirectionalLRUMap.Node node = backingStore.indexToNode[backingStore.currSize]; + if (node != null) { + saveIfNeeded(node); + } + saveIfNeeded(backingStore.head); + } else { + BidirectionalLRUMap.Node tail = backingStore.tail; + saveIfNeeded(tail); + if (tail != null && tail.prev != null) { + saveIfNeeded(tail.prev); + } + saveIfNeeded(backingStore.head); + } + } + return addEntryInternal(data, offset, length, true); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/TraceUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/TraceUtil.java index 5b1fb86a351a..260c0064f840 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/TraceUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/TraceUtil.java @@ -28,8 +28,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; -import org.apache.hadoop.hbase.Version; import org.apache.hadoop.hbase.util.FutureUtils; +import org.apache.hadoop.hbase.util.VersionInfo; import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private @@ -39,7 +39,7 @@ private TraceUtil() { } public static Tracer getGlobalTracer() { - return GlobalOpenTelemetry.getTracer("org.apache.hbase", Version.version); + return GlobalOpenTelemetry.getTracer("org.apache.hbase", VersionInfo.getVersion()); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java index 73bb6f38cd27..26ed7c982581 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java @@ -188,11 +188,39 @@ public static int getDefaultBufferSize(final FileSystem fs) { */ public static FSDataOutputStream create(FileSystem fs, Path path, FsPermission perm, boolean overwrite) throws IOException { + return create(fs, path, perm, overwrite, true); + } + + /** + * Create the specified file on the filesystem. By default, this will: + *

    + *
  1. apply the umask in the configuration (if it is enabled)
    2. + *
  2. use the fs configured buffer size (or 4096 if not set)
    3. + *
  3. use the default replication
    4. + *
  4. use the default block size
    5. + *
  5. not track progress
    6. + *
+ * @param fs {@link FileSystem} on which to write the file + * @param path {@link Path} to the file to write + * @param perm intial permissions + * @param overwrite Whether or not the created file should be overwritten. + * @param isRecursiveCreate recursively create parent directories + * @return output stream to the created file + * @throws IOException if the file cannot be created + */ + public static FSDataOutputStream create(FileSystem fs, Path path, FsPermission perm, + boolean overwrite, boolean isRecursiveCreate) throws IOException { if (LOG.isTraceEnabled()) { - LOG.trace("Creating file={} with permission={}, overwrite={}", path, perm, overwrite); + LOG.trace("Creating file={} with permission={}, overwrite={}, recursive={}", path, perm, + overwrite, isRecursiveCreate); + } + if (isRecursiveCreate) { + return fs.create(path, perm, overwrite, getDefaultBufferSize(fs), + getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null); + } else { + return fs.createNonRecursive(path, perm, overwrite, getDefaultBufferSize(fs), + getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null); } - return fs.create(path, perm, overwrite, getDefaultBufferSize(fs), - getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastStringPool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastStringPool.java new file mode 100644 index 000000000000..a5f0dc17bcc1 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastStringPool.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.lang3.mutable.MutableObject; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; + +/** + * A string pool like {@link String#intern()}, but more flexible as we can create multiple instances + * and use them in difference places, where {@link String#intern()} is global. + *

+ * We use {@link WeakReference} so when there are no actual reference to the String, it will be GCed + * to reduce memory pressure. + *

+ * The difference between {@link WeakObjectPool} is that, we also need to use {@link WeakReference} + * as key, not only value, because the key(a String) is exactly what we want to deduplicate. + */ +@InterfaceAudience.Private +public class FastStringPool { + + private static final class WeakKey extends WeakReference { + + private final int hash; + + WeakKey(String referent, ReferenceQueue queue) { + super(Preconditions.checkNotNull(referent), queue); + // must calculate it here, as later the referent may be GCed + this.hash = referent.hashCode(); + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof WeakKey)) { + return false; + } + + String a = this.get(); + String b = ((WeakKey) obj).get(); + // In ConcurrentHashMap, we will always compare references(like entry.key == key) before + // calling actual equals method, so this will not cause problems for clean up. And in normal + // intern path, the reference will never be null, so there is no problem too. + if (a == null || b == null) { + return false; + } + return a.equals(b); + } + } + + private final ConcurrentHashMap> map = new ConcurrentHashMap<>(); + + private final ReferenceQueue refQueue = new ReferenceQueue<>(); + + private final Lock cleanupLock = new ReentrantLock(); + + // only call cleanup every 256 times + private static final int CLEANUP_MASK = 0xFF; + private final AtomicInteger counter = new AtomicInteger(); + + public String intern(String s) { + Preconditions.checkNotNull(s); + maybeCleanup(); + + WeakKey lookupKey = new WeakKey(s, null); + WeakReference ref = map.get(lookupKey); + if (ref != null) { + String v = ref.get(); + if (v != null) { + return v; + } + } + + WeakKey storeKey = new WeakKey(s, refQueue); + WeakReference storeVal = new WeakReference<>(s); + // Used to store the return value. The return value of compute method is a WeakReference, the + // value of the WeakReference may be GCed just before we get it for returning. + MutableObject ret = new MutableObject<>(); + + map.compute(storeKey, (k, prevVal) -> { + if (prevVal == null) { + ret.setValue(s); + return storeVal; + } else { + String prevRef = prevVal.get(); + if (prevRef != null) { + ret.setValue(prevRef); + return prevVal; + } else { + ret.setValue(s); + return storeVal; + } + } + }); + assert ret.get() != null; + return ret.get(); + } + + private void cleanup() { + if (!cleanupLock.tryLock()) { + // a cleanup task is ongoing, give up + return; + } + try { + for (;;) { + WeakKey k = (WeakKey) refQueue.poll(); + if (k == null) { + return; + } + map.remove(k); + } + } finally { + cleanupLock.unlock(); + } + } + + private void maybeCleanup() { + if ((counter.incrementAndGet() & CLEANUP_MASK) != 0) { + return; + } + cleanup(); + } + + public int size() { + // size method is not on critical path, so always call cleanup here to reduce memory pressure + cleanup(); + return map.size(); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java index 60a37af60483..80ae42a3a3b9 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java @@ -40,12 +40,15 @@ public class VersionInfo { // higher than any numbers in the version. private static final int VERY_LARGE_NUMBER = 100000; + // Copying into a non-final member so that it can be changed by reflection for testing + private static String version = Version.version; + /** * Get the hbase version. * @return the hbase version string, eg. "0.6.3-dev" */ public static String getVersion() { - return Version.version; + return version; } /** diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 15b970c7e8d0..2f7c1b514a13 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -288,6 +288,15 @@ possible configurations would overwhelm and obscure the important. org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter The WAL file writer implementation. + + hbase.regionserver.free.heap.min.memory.size + + Defines the minimum amount of heap memory that must remain free for the RegionServer to start, + specified in bytes or human-readable formats like '512m' for megabytes or '4g' for gigabytes. + If not set, the default is 20% of the total heap size. To disable the check entirely, + set this value to 0. If the combined memory usage of memstore and block cache + exceeds (total heap - this value), the RegionServer will fail to start. + hbase.regionserver.global.memstore.size @@ -299,6 +308,16 @@ possible configurations would overwhelm and obscure the important. honor the old hbase.regionserver.global.memstore.upperLimit property if present. + + hbase.regionserver.global.memstore.memory.size + + Maximum size of all memstores in a region server before new + updates are blocked and flushes are forced, specified in bytes or human-readable formats + like '10m' for megabytes or '10g' for gigabytes. This configuration allows setting + an absolute memory size instead of a percentage of the maximum heap. Takes precedence + over hbase.regionserver.global.memstore.size if both are specified. + + hbase.regionserver.global.memstore.size.lower.limit diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java index 1bc648aeb0b5..dc51187e3cf8 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java @@ -19,9 +19,11 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Pattern; -import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; import org.junit.runners.Suite; /** @@ -46,6 +48,16 @@ public static Class[] getCategoryAnnotations(Class c) { return new Class[0]; } + public static String[] getTagAnnotations(Class c) { + // TODO handle optional Tags annotation + Tag[] tags = c.getAnnotationsByType(Tag.class); + List values = new ArrayList<>(); + for (Tag tag : tags) { + values.add(tag.value()); + } + return values.toArray(new String[values.size()]); + } + /** Filters both test classes and anything in the hadoop-compat modules */ public static class TestFileNameFilter implements FileNameFilter, ResourcePathFilter { private static final Pattern hadoopCompactRe = Pattern.compile("hbase-hadoop\\d?-compat"); @@ -92,7 +104,10 @@ private boolean isTestClass(Class c) { } for (Method met : c.getMethods()) { - if (met.getAnnotation(Test.class) != null) { + if ( + met.getAnnotation(org.junit.Test.class) != null + || met.getAnnotation(org.junit.jupiter.api.Test.class) != null + ) { return true; } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java new file mode 100644 index 000000000000..9d4ea87e0ec1 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.hadoop.hbase.testclassification.IntegrationTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.yetus.audience.InterfaceAudience; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Store; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.junit.platform.commons.JUnitException; +import org.junit.platform.commons.util.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Class test rule implementation for JUnit5. + *

+ * It ensures that all JUnit5 tests should have at least one of {@link SmallTests}, + * {@link MediumTests}, {@link LargeTests}, {@link IntegrationTests} tags, and set timeout based on + * the tag. + *

+ * It also controls the timeout for the whole test class running, while the timeout annotation in + * JUnit5 can only enforce the timeout for each test method. When a test is timed out, a thread dump + * will be printed to log output. + *

+ * It also implements resource check for each test method, using the {@link ResourceChecker} class. + *

+ * Finally, it also forbid System.exit call in tests.
+ * TODO: need to find a new way as SecurityManager was deprecated in Java 17 and permanently + * disabled since Java 24. + */ +@InterfaceAudience.Private +public class HBaseJupiterExtension implements InvocationInterceptor, BeforeAllCallback, + AfterAllCallback, BeforeEachCallback, AfterEachCallback { + + private static final Logger LOG = LoggerFactory.getLogger(HBaseJupiterExtension.class); + + private static final SecurityManager securityManager = new TestSecurityManager(); + + private static final ExtensionContext.Namespace NAMESPACE = + ExtensionContext.Namespace.create(HBaseJupiterExtension.class); + + private static final Map TAG_TO_TIMEOUT = + ImmutableMap.of(SmallTests.TAG, Duration.ofMinutes(3), MediumTests.TAG, Duration.ofMinutes(6), + LargeTests.TAG, Duration.ofMinutes(13), IntegrationTests.TAG, Duration.ZERO); + + private static final String EXECUTOR = "executor"; + + private static final String DEADLINE = "deadline"; + + private static final String RESOURCE_CHECK = "rc"; + + private Duration pickTimeout(ExtensionContext ctx) { + Set timeoutTags = TAG_TO_TIMEOUT.keySet(); + Set timeoutTag = Sets.intersection(timeoutTags, ctx.getTags()); + if (timeoutTag.isEmpty()) { + fail("Test class " + ctx.getDisplayName() + " does not have any of the following scale tags " + + timeoutTags); + } + if (timeoutTag.size() > 1) { + fail("Test class " + ctx.getDisplayName() + " has multiple scale tags " + timeoutTag); + } + return TAG_TO_TIMEOUT.get(Iterables.getOnlyElement(timeoutTag)); + } + + @Override + public void beforeAll(ExtensionContext ctx) throws Exception { + // TODO: remove this usage + System.setSecurityManager(securityManager); + Duration timeout = pickTimeout(ctx); + if (timeout.isZero() || timeout.isNegative()) { + LOG.info("No timeout for {}", ctx.getDisplayName()); + // zero means no timeout + return; + } + Instant deadline = Instant.now().plus(timeout); + LOG.info("Timeout for {} is {}, it should be finished before {}", ctx.getDisplayName(), timeout, + deadline); + ExecutorService executor = + Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("HBase-Test-" + ctx.getDisplayName() + "-Main-Thread").build()); + Store store = ctx.getStore(NAMESPACE); + store.put(EXECUTOR, executor); + store.put(DEADLINE, deadline); + } + + @Override + public void afterAll(ExtensionContext ctx) throws Exception { + Store store = ctx.getStore(NAMESPACE); + ExecutorService executor = store.remove(EXECUTOR, ExecutorService.class); + if (executor != null) { + executor.shutdownNow(); + } + store.remove(DEADLINE); + // reset secutiry manager + System.setSecurityManager(null); + } + + private T runWithTimeout(Invocation invocation, ExtensionContext ctx, String name) + throws Throwable { + Store store = ctx.getStore(NAMESPACE); + ExecutorService executor = store.get(EXECUTOR, ExecutorService.class); + if (executor == null) { + return invocation.proceed(); + } + Instant deadline = store.get(DEADLINE, Instant.class); + Instant now = Instant.now(); + if (!now.isBefore(deadline)) { + fail("Test " + name + " timed out, deadline is " + deadline); + return null; + } + + Duration remaining = Duration.between(now, deadline); + LOG.info("remaining timeout for {} is {}", name, remaining); + Future future = executor.submit(() -> { + try { + return invocation.proceed(); + } catch (Throwable t) { + // follow the same pattern with junit5 + throw ExceptionUtils.throwAsUncheckedException(t); + } + }); + try { + return future.get(remaining.toNanos(), TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("Test " + name + " interrupted"); + return null; + } catch (ExecutionException e) { + throw ExceptionUtils.throwAsUncheckedException(e.getCause()); + } catch (TimeoutException e) { + printThreadDump(); + throw new JUnitException("Test " + name + " timed out, deadline is " + deadline, e); + } + } + + private void printThreadDump() { + LOG.info("====> TEST TIMED OUT. PRINTING THREAD DUMP. <===="); + LOG.info(TimedOutTestsListener.buildThreadDiagnosticString()); + } + + @Override + public void interceptBeforeAllMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".beforeAll"); + } + + @Override + public void interceptBeforeEachMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".beforeEach"); + } + + @Override + public void interceptTestMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName()); + } + + @Override + public void interceptAfterEachMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".afterEach"); + } + + @Override + public void interceptAfterAllMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".afterAll"); + } + + @Override + public T interceptTestClassConstructor(Invocation invocation, + ReflectiveInvocationContext> invocationContext, + ExtensionContext extensionContext) throws Throwable { + return runWithTimeout(invocation, extensionContext, + extensionContext.getDisplayName() + ".constructor"); + } + + // below are for implementing resource checker around test method + + @Override + public void beforeEach(ExtensionContext ctx) throws Exception { + ResourceChecker rc = new ResourceChecker(ctx.getDisplayName()); + JUnitResourceCheckers.addResourceAnalyzer(rc); + Store store = ctx.getStore(NAMESPACE); + store.put(RESOURCE_CHECK, rc); + rc.start(); + } + + @Override + public void afterEach(ExtensionContext ctx) throws Exception { + Store store = ctx.getStore(NAMESPACE); + ResourceChecker rc = store.remove(RESOURCE_CHECK, ResourceChecker.class); + if (rc != null) { + rc.end(); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedInvocationContext.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedInvocationContext.java new file mode 100644 index 000000000000..e13c7e58c801 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedInvocationContext.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.extension.Extension; +import org.junit.jupiter.api.extension.TestTemplateInvocationContext; +import org.junit.jupiter.params.provider.Arguments; + +/** + * @see HBaseParameterizedTestTemplate + */ +public class HBaseParameterizedInvocationContext implements TestTemplateInvocationContext { + + private final Arguments arguments; + + private final String namePattern; + + HBaseParameterizedInvocationContext(Arguments arguments, String namePattern) { + this.arguments = arguments; + this.namePattern = namePattern; + } + + @Override + public String getDisplayName(int invocationIndex) { + String result = namePattern.replace("{index}", String.valueOf(invocationIndex)); + + Object[] args = arguments.get(); + for (int i = 0; i < args.length; i++) { + result = result.replace("{" + i + "}", String.valueOf(args[i])); + } + return result; + } + + @Override + public List getAdditionalExtensions() { + return Collections.singletonList(new HBaseParameterizedParameterResolver(arguments)); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedParameterResolver.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedParameterResolver.java new file mode 100644 index 000000000000..1df5b22a7029 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedParameterResolver.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.jupiter.params.provider.Arguments; + +import org.apache.hbase.thirdparty.com.google.common.primitives.Primitives; + +/** + * @see HBaseParameterizedTestTemplate + */ +public class HBaseParameterizedParameterResolver implements ParameterResolver { + + private final Object[] values; + + HBaseParameterizedParameterResolver(Arguments arguments) { + this.values = arguments.get(); + } + + @Override + public boolean supportsParameter(ParameterContext pc, ExtensionContext ec) + throws ParameterResolutionException { + int index = pc.getIndex(); + if (index >= values.length) { + return false; + } + Object value = values[index]; + Class expectedType = pc.getParameter().getType(); + if (expectedType.isPrimitive()) { + // primitive type can not accept null value + if (value == null) { + return false; + } + // test with wrapper type, otherwise it will always return false + return Primitives.wrap(expectedType).isAssignableFrom(value.getClass()); + } + return expectedType.isAssignableFrom(value.getClass()); + } + + @Override + public Object resolveParameter(ParameterContext pc, ExtensionContext ec) + throws ParameterResolutionException { + return values[pc.getIndex()]; + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTemplateProvider.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTemplateProvider.java new file mode 100644 index 000000000000..01632233f310 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTemplateProvider.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.stream.Stream; +import org.apache.yetus.audience.InterfaceAudience; +import org.junit.jupiter.api.extension.ExtensionConfigurationException; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.TestTemplateInvocationContext; +import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider; +import org.junit.jupiter.params.provider.Arguments; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The entry point class for supporting JUnit4 like Parameterized test, where we can use constructor + * to pass parameters. + *

+ * JUnit5's {@link org.junit.jupiter.params.ParameterizedClass} will create separated test classes, + * which is different with JUnit4 and {@link org.junit.jupiter.params.ParameterizedTest} does not + * support passing parameters through constructors. + *

+ * When you want to use this provider, annotation the test class with + * {@link HBaseParameterizedTestTemplate}, and provide a static method named "parameters" for + * providing the arguments. The method must have no parameter, and return a Stream<Arguments>. + * All the test method should be marked with {@link org.junit.jupiter.api.TestTemplate}, not + * {@link org.junit.jupiter.api.Test} or {@link org.junit.jupiter.params.ParameterizedTest}. + * @see HBaseParameterizedTestTemplate + * @see HBaseParameterizedInvocationContext + * @see HBaseParameterizedParameterResolver + */ +@InterfaceAudience.Private +public class HBaseParameterizedTemplateProvider implements TestTemplateInvocationContextProvider { + + private static final Logger LOG = + LoggerFactory.getLogger(HBaseParameterizedTemplateProvider.class); + + private static final String PARAMETERS_METHOD_NAME = "parameters"; + + @Override + public boolean supportsTestTemplate(ExtensionContext context) { + return context.getTestClass() + .map(c -> c.isAnnotationPresent(HBaseParameterizedTestTemplate.class)).orElse(false); + } + + private Method findParametersMethod(final Class testClass) { + Class clazz = testClass; + for (;;) { + try { + return clazz.getDeclaredMethod(PARAMETERS_METHOD_NAME); + } catch (NoSuchMethodException e) { + Class parentClass = clazz.getSuperclass(); + LOG.debug("Can not find method {} in class {}, try its parent class {}", + PARAMETERS_METHOD_NAME, clazz, parentClass); + if (parentClass == null) { + throw new ExtensionConfigurationException("Can not find a static " + + PARAMETERS_METHOD_NAME + " method in class " + testClass + "or its super classes"); + } + clazz = parentClass; + } + } + + } + + @Override + public Stream + provideTestTemplateInvocationContexts(ExtensionContext context) { + Class testClass = context.getRequiredTestClass(); + // get parameters + Method method = findParametersMethod(testClass); + if (!Modifier.isStatic(method.getModifiers())) { + throw new ExtensionConfigurationException(PARAMETERS_METHOD_NAME + " method must be static"); + } + if (method.getParameterCount() > 0) { + throw new ExtensionConfigurationException( + PARAMETERS_METHOD_NAME + " method must not have any parameters"); + } + + Stream args; + try { + args = (Stream) method.invoke(null); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new ExtensionConfigurationException("failed to invoke parameters method", e); + } + // get display name + String namePattern = testClass.getAnnotation(HBaseParameterizedTestTemplate.class).name(); + + return args.map(arg -> new HBaseParameterizedInvocationContext(arg, namePattern)); + } + +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTestTemplate.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTestTemplate.java new file mode 100644 index 000000000000..425869f98e0c --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTestTemplate.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.apache.yetus.audience.InterfaceAudience; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * The annotation for running JUnit4 like parameterized tests with JUnit5. + * @see HBaseParameterizedTestTemplate + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(HBaseParameterizedTemplateProvider.class) +@InterfaceAudience.Private +public @interface HBaseParameterizedTestTemplate { + + String name() default "[{index}]"; +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java new file mode 100644 index 000000000000..aee49dc2c60f --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hbase.ResourceChecker.Phase; +import org.apache.hadoop.hbase.util.JVM; + +/** + * ResourceCheckers when running JUnit tests. + */ +public final class JUnitResourceCheckers { + + private JUnitResourceCheckers() { + } + + private static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + private Set initialThreadNames = new HashSet<>(); + private List stringsToLog = null; + + @Override + public int getVal(Phase phase) { + Map stackTraces = Thread.getAllStackTraces(); + if (phase == Phase.INITIAL) { + stringsToLog = null; + for (Thread t : stackTraces.keySet()) { + initialThreadNames.add(t.getName()); + } + } else if (phase == Phase.END) { + if (stackTraces.size() > initialThreadNames.size()) { + stringsToLog = new ArrayList<>(); + for (Thread t : stackTraces.keySet()) { + if (!initialThreadNames.contains(t.getName())) { + stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); + StackTraceElement[] stackElements = stackTraces.get(t); + for (StackTraceElement ele : stackElements) { + stringsToLog.add("\t" + ele + "\n"); + } + } + } + } + } + return stackTraces.size(); + } + + @Override + public int getMax() { + return 500; + } + + @Override + public List getStringsToLog() { + return stringsToLog; + } + } + + private static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + JVM jvm = new JVM(); + return (int) jvm.getOpenFileDescriptorCount(); + } + + @Override + public int getMax() { + return 1024; + } + } + + private static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + JVM jvm = new JVM(); + return (int) jvm.getMaxFileDescriptorCount(); + } + } + + private static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return (int) (new JVM().getSystemLoadAverage() * 100); + } + } + + private static class ProcessCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return new JVM().getNumberOfRunningProcess(); + } + } + + private static class AvailableMemoryMBResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return (int) (new JVM().getFreeMemory() / (1024L * 1024L)); + } + } + + public static void addResourceAnalyzer(ResourceChecker rc) { + rc.addResourceAnalyzer(new ThreadResourceAnalyzer()); + rc.addResourceAnalyzer(new OpenFileDescriptorResourceAnalyzer()); + rc.addResourceAnalyzer(new MaxFileDescriptorResourceAnalyzer()); + rc.addResourceAnalyzer(new SystemLoadAverageResourceAnalyzer()); + rc.addResourceAnalyzer(new ProcessCountResourceAnalyzer()); + rc.addResourceAnalyzer(new AvailableMemoryMBResourceAnalyzer()); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java index 4dfce7f536b5..2a796cc40774 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java @@ -17,14 +17,8 @@ */ package org.apache.hadoop.hbase; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hadoop.hbase.ResourceChecker.Phase; -import org.apache.hadoop.hbase.util.JVM; import org.junit.runner.notification.RunListener; /** @@ -38,104 +32,8 @@ * When surefire forkMode=once/always/perthread, this code is executed on the forked process. */ public class ResourceCheckerJUnitListener extends RunListener { - private Map rcs = new ConcurrentHashMap<>(); - static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - private static Set initialThreadNames = new HashSet<>(); - private static List stringsToLog = null; - - @Override - public int getVal(Phase phase) { - Map stackTraces = Thread.getAllStackTraces(); - if (phase == Phase.INITIAL) { - stringsToLog = null; - for (Thread t : stackTraces.keySet()) { - initialThreadNames.add(t.getName()); - } - } else if (phase == Phase.END) { - if (stackTraces.size() > initialThreadNames.size()) { - stringsToLog = new ArrayList<>(); - for (Thread t : stackTraces.keySet()) { - if (!initialThreadNames.contains(t.getName())) { - stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); - StackTraceElement[] stackElements = stackTraces.get(t); - for (StackTraceElement ele : stackElements) { - stringsToLog.add("\t" + ele + "\n"); - } - } - } - } - } - return stackTraces.size(); - } - - @Override - public int getMax() { - return 500; - } - - @Override - public List getStringsToLog() { - return stringsToLog; - } - } - - static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - JVM jvm = new JVM(); - return (int) jvm.getOpenFileDescriptorCount(); - } - - @Override - public int getMax() { - return 1024; - } - } - - static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - JVM jvm = new JVM(); - return (int) jvm.getMaxFileDescriptorCount(); - } - } - - static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return (int) (new JVM().getSystemLoadAverage() * 100); - } - } - - static class ProcessCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return new JVM().getNumberOfRunningProcess(); - } - } - - static class AvailableMemoryMBResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return (int) (new JVM().getFreeMemory() / (1024L * 1024L)); - } - } + private final Map rcs = new ConcurrentHashMap<>(); /** * To be implemented by sub classes if they want to add specific ResourceAnalyzer. @@ -145,17 +43,9 @@ protected void addResourceAnalyzer(ResourceChecker rc) { private void start(String testName) { ResourceChecker rc = new ResourceChecker(testName); - rc.addResourceAnalyzer(new ThreadResourceAnalyzer()); - rc.addResourceAnalyzer(new OpenFileDescriptorResourceAnalyzer()); - rc.addResourceAnalyzer(new MaxFileDescriptorResourceAnalyzer()); - rc.addResourceAnalyzer(new SystemLoadAverageResourceAnalyzer()); - rc.addResourceAnalyzer(new ProcessCountResourceAnalyzer()); - rc.addResourceAnalyzer(new AvailableMemoryMBResourceAnalyzer()); - + JUnitResourceCheckers.addResourceAnalyzer(rc); addResourceAnalyzer(rc); - rcs.put(testName, rc); - rc.start(); } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestBuildThreadDiagnosticString.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestBuildThreadDiagnosticString.java new file mode 100644 index 000000000000..4071f18e2dd6 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestBuildThreadDiagnosticString.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; + +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestBuildThreadDiagnosticString { + + @Test + public void test() { + String threadDump = TimedOutTestsListener.buildThreadDiagnosticString(); + System.out.println(threadDump); + assertThat(threadDump, + containsString(getClass().getName() + ".test(" + getClass().getSimpleName() + ".java:")); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java new file mode 100644 index 000000000000..3e30b388ab2e --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.Field; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * Verify that the values are all correct. + */ +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestJUnit5TagConstants { + + @Test + public void testVerify() throws Exception { + ClassFinder finder = new ClassFinder(getClass().getClassLoader()); + for (Class annoClazz : finder.findClasses(ClientTests.class.getPackage().getName(), false)) { + Field field = annoClazz.getField("TAG"); + assertEquals(annoClazz.getName(), field.get(null)); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TimedOutTestsListener.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TimedOutTestsListener.java index 00860d0dde58..253d17359778 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TimedOutTestsListener.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TimedOutTestsListener.java @@ -26,9 +26,9 @@ import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.nio.charset.StandardCharsets; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.Map; import org.junit.runner.notification.Failure; @@ -40,7 +40,10 @@ */ public class TimedOutTestsListener extends RunListener { - static final String TEST_TIMED_OUT_PREFIX = "test timed out after"; + private static final String TEST_TIMED_OUT_PREFIX = "test timed out after"; + + private static final DateTimeFormatter TIMESTAMP_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS Z").withZone(ZoneId.systemDefault()); private static String INDENT = " "; @@ -67,13 +70,11 @@ public void testFailure(Failure failure) throws Exception { output.flush(); } - @SuppressWarnings("JavaUtilDate") public static String buildThreadDiagnosticString() { StringWriter sw = new StringWriter(); PrintWriter output = new PrintWriter(sw); - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); - output.println(String.format("Timestamp: %s", dateFormat.format(new Date()))); + output.println(String.format("Timestamp: %s", TIMESTAMP_FORMATTER.format(Instant.now()))); output.println(); output.println(buildThreadDump()); @@ -87,7 +88,7 @@ public static String buildThreadDiagnosticString() { return sw.toString(); } - static String buildThreadDump() { + private static String buildThreadDump() { StringBuilder dump = new StringBuilder(); Map stackTraces = Thread.getAllStackTraces(); for (Map.Entry e : stackTraces.entrySet()) { @@ -109,7 +110,7 @@ static String buildThreadDump() { return dump.toString(); } - static String buildDeadlockInfo() { + private static String buildDeadlockInfo() { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); long[] threadIds = threadBean.findMonitorDeadlockedThreads(); if (threadIds != null && threadIds.length > 0) { diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestFileChangeWatcher.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestFileChangeWatcher.java index c24e96f8da6b..ff906e3bb834 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestFileChangeWatcher.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestFileChangeWatcher.java @@ -17,12 +17,14 @@ */ package org.apache.hadoop.hbase.io; +import static org.awaitility.Awaitility.await; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.endsWith; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import java.io.File; import java.io.IOException; @@ -30,22 +32,23 @@ import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.io.crypto.tls.X509Util; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.hamcrest.Matchers; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,31 +58,51 @@ * "https://github.com/apache/zookeeper/blob/391cb4aa6b54e19a028215e1340232a114c23ed3/zookeeper-server/src/test/java/org/apache/zookeeper/common/FileChangeWatcherTest.java">Base * revision */ -@Category({ IOTests.class, SmallTests.class }) +@Tag(IOTests.TAG) +@Tag(SmallTests.TAG) public class TestFileChangeWatcher { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFileChangeWatcher.class); - - private static File tempFile; - private static final Logger LOG = LoggerFactory.getLogger(TestFileChangeWatcher.class); private static final HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility(); - private static final long FS_TIMEOUT = 30000L; + private static File dir; + private static final Duration POLL_INTERVAL = Duration.ofMillis(100); - @BeforeClass - public static void createTempFile() throws IOException { - tempFile = File.createTempFile("zk_test_", ""); + private File tempFile; + + private FileChangeWatcher watcher; + + @BeforeAll + public static void setUpBeforeAll() throws IOException { + dir = new File(UTIL.getDataTestDir().toString()).getAbsoluteFile(); + if (!dir.mkdirs()) { + throw new IOException("can not mkdir " + dir); + } } - @AfterClass + @AfterAll public static void cleanupTempDir() { UTIL.cleanupTestDir(); } + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException { + tempFile = new File(dir, "file_change_test_" + testInfo.getDisplayName()); + if (!tempFile.createNewFile()) { + throw new IOException("failed to create new empty file " + tempFile); + } + } + + @AfterEach + public void tearDown() throws InterruptedException { + if (watcher != null) { + watcher.stop(); + watcher.waitForState(FileChangeWatcher.State.STOPPED); + watcher = null; + } + } + @Test public void testEnableCertFileReloading() throws IOException { Configuration myConf = new Configuration(); @@ -91,7 +114,7 @@ public void testEnableCertFileReloading() throws IOException { X509Util.enableCertFileReloading(myConf, keystoreWatcher, truststoreWatcher, () -> { }); assertNotNull(keystoreWatcher.get()); - assertThat(keystoreWatcher.get().getWatcherThreadName(), Matchers.endsWith("foo.jks")); + assertThat(keystoreWatcher.get().getWatcherThread().getName(), endsWith("foo.jks")); assertNull(truststoreWatcher.get()); keystoreWatcher.getAndSet(null).stop(); @@ -103,149 +126,97 @@ public void testEnableCertFileReloading() throws IOException { }); assertNotNull(keystoreWatcher.get()); - assertThat(keystoreWatcher.get().getWatcherThreadName(), Matchers.endsWith("foo.jks")); + assertThat(keystoreWatcher.get().getWatcherThread().getName(), endsWith("foo.jks")); assertNotNull(truststoreWatcher.get()); - assertThat(truststoreWatcher.get().getWatcherThreadName(), Matchers.endsWith("bar.jks")); + assertThat(truststoreWatcher.get().getWatcherThread().getName(), endsWith("bar.jks")); keystoreWatcher.getAndSet(null).stop(); truststoreWatcher.getAndSet(null).stop(); } + // wait until watcher thread finish loading the last modified time, we check this by checking + // whether the watcher thread has been in TIMED_WAITING state, i.e, waiting for the next runLoop + private void awaitWatcherThreadInitialized() throws InterruptedException { + watcher.waitForState(FileChangeWatcher.State.RUNNING); + await().atMost(Duration.ofSeconds(2)).pollInSameThread().pollInterval(Duration.ofMillis(10)) + .until(() -> watcher.getWatcherThread().getState() == Thread.State.TIMED_WAITING); + } + @Test - public void testNoFalseNotifications() throws IOException, InterruptedException { - FileChangeWatcher watcher = null; - try { - final List notifiedPaths = new ArrayList<>(); - watcher = new FileChangeWatcher(tempFile.toPath(), "test", POLL_INTERVAL, path -> { - LOG.info("Got an update on path {}", path); - synchronized (notifiedPaths) { - notifiedPaths.add(path); - notifiedPaths.notifyAll(); - } - }); - watcher.start(); - watcher.waitForState(FileChangeWatcher.State.RUNNING); - Thread.sleep(1000L); // TODO hack - assertEquals("Should not have been notified", 0, notifiedPaths.size()); - } finally { - if (watcher != null) { - watcher.stop(); - watcher.waitForState(FileChangeWatcher.State.STOPPED); + public void testNoFalseNotifications() throws Exception { + final List notifiedPaths = new ArrayList<>(); + watcher = new FileChangeWatcher(tempFile.toPath(), "test", POLL_INTERVAL, path -> { + LOG.info("Got an update on path {}", path); + synchronized (notifiedPaths) { + notifiedPaths.add(path); + notifiedPaths.notifyAll(); } - } + }); + watcher.start(); + awaitWatcherThreadInitialized(); + await().during(Duration.ofSeconds(2)).atMost(Duration.ofSeconds(3)) + .untilAsserted(() -> assertEquals("Should not have been notified", 0, notifiedPaths.size())); } @Test public void testCallbackWorksOnFileChanges() throws IOException, InterruptedException { - FileChangeWatcher watcher = null; - try { - final List notifiedPaths = new ArrayList<>(); - watcher = new FileChangeWatcher(tempFile.toPath(), "test", POLL_INTERVAL, path -> { - LOG.info("Got an update on path {}", path); - synchronized (notifiedPaths) { - notifiedPaths.add(path); - notifiedPaths.notifyAll(); - } - }); - watcher.start(); - watcher.waitForState(FileChangeWatcher.State.RUNNING); - Thread.sleep(1000L); // TODO hack - for (int i = 0; i < 3; i++) { - LOG.info("Modifying file, attempt {}", (i + 1)); - FileUtils.writeStringToFile(tempFile, "Hello world " + i + "\n", StandardCharsets.UTF_8, - true); - synchronized (notifiedPaths) { - if (notifiedPaths.size() < i + 1) { - notifiedPaths.wait(FS_TIMEOUT); - } - assertEquals("Wrong number of notifications", i + 1, notifiedPaths.size()); - Path path = notifiedPaths.get(i); - assertEquals(tempFile.getPath(), path.toString()); - } - } - } finally { - if (watcher != null) { - watcher.stop(); - watcher.waitForState(FileChangeWatcher.State.STOPPED); - } + final List notifiedPaths = Collections.synchronizedList(new ArrayList<>()); + watcher = new FileChangeWatcher(tempFile.toPath(), "test", POLL_INTERVAL, path -> { + LOG.info("Got an update on path {}", path); + notifiedPaths.add(path); + }); + watcher.start(); + awaitWatcherThreadInitialized(); + for (int i = 0; i < 3; i++) { + final int index = i; + LOG.info("Modifying file, attempt {}", (index + 1)); + FileUtils.writeStringToFile(tempFile, "Hello world " + index + "\n", StandardCharsets.UTF_8, + true); + await().atMost(Duration.ofSeconds(2)).untilAsserted( + () -> assertEquals("Wrong number of notifications", index + 1, notifiedPaths.size())); + Path path = notifiedPaths.get(index); + assertEquals(tempFile.getPath(), path.toString()); } } @Test public void testCallbackWorksOnFileTouched() throws IOException, InterruptedException { - FileChangeWatcher watcher = null; - try { - final List notifiedPaths = new ArrayList<>(); - watcher = new FileChangeWatcher(tempFile.toPath(), "test", POLL_INTERVAL, path -> { - LOG.info("Got an update on path {}", path); - synchronized (notifiedPaths) { - notifiedPaths.add(path); - notifiedPaths.notifyAll(); - } - }); - watcher.start(); - watcher.waitForState(FileChangeWatcher.State.RUNNING); - Thread.sleep(1000L); // TODO hack - LOG.info("Touching file"); - FileUtils.touch(tempFile); - synchronized (notifiedPaths) { - if (notifiedPaths.isEmpty()) { - notifiedPaths.wait(FS_TIMEOUT); - } - assertFalse(notifiedPaths.isEmpty()); - Path path = notifiedPaths.get(0); - assertEquals(tempFile.getPath(), path.toString()); - } - } finally { - if (watcher != null) { - watcher.stop(); - watcher.waitForState(FileChangeWatcher.State.STOPPED); - } - } + final List notifiedPaths = Collections.synchronizedList(new ArrayList<>()); + watcher = new FileChangeWatcher(tempFile.toPath(), "test", POLL_INTERVAL, path -> { + LOG.info("Got an update on path {}", path); + notifiedPaths.add(path); + }); + watcher.start(); + awaitWatcherThreadInitialized(); + LOG.info("Touching file"); + FileUtils.touch(tempFile); + await().atMost(Duration.ofSeconds(2)).untilAsserted(() -> assertFalse(notifiedPaths.isEmpty())); + Path path = notifiedPaths.get(0); + assertEquals(tempFile.getPath(), path.toString()); } @Test - public void testCallbackErrorDoesNotCrashWatcherThread() - throws IOException, InterruptedException { - FileChangeWatcher watcher = null; - try { - final AtomicInteger callCount = new AtomicInteger(0); - watcher = new FileChangeWatcher(tempFile.toPath(), "test", POLL_INTERVAL, path -> { - LOG.info("Got an update for path {}", path); - int oldValue; - synchronized (callCount) { - oldValue = callCount.getAndIncrement(); - callCount.notifyAll(); - } - if (oldValue == 0) { - throw new RuntimeException("This error should not crash the watcher thread"); - } - }); - watcher.start(); - watcher.waitForState(FileChangeWatcher.State.RUNNING); - Thread.sleep(1000L); // TODO hack - LOG.info("Modifying file"); - FileUtils.writeStringToFile(tempFile, "Hello world\n", StandardCharsets.UTF_8, true); - synchronized (callCount) { - while (callCount.get() == 0) { - callCount.wait(FS_TIMEOUT); - } - } - LOG.info("Modifying file again"); - FileUtils.writeStringToFile(tempFile, "Hello world again\n", StandardCharsets.UTF_8, true); - synchronized (callCount) { - if (callCount.get() == 1) { - callCount.wait(FS_TIMEOUT); - } - } - // The value of callCount can exceed 1 only if the callback thread - // survives the exception thrown by the first callback. - assertTrue(callCount.get() > 1); - } finally { - if (watcher != null) { - watcher.stop(); - watcher.waitForState(FileChangeWatcher.State.STOPPED); - } - } + public void testCallbackErrorDoesNotCrashWatcherThread() throws Exception { + final AtomicInteger callCount = new AtomicInteger(0); + watcher = new FileChangeWatcher(tempFile.toPath(), "test", POLL_INTERVAL, path -> { + LOG.info("Got an update for path {}", path); + callCount.incrementAndGet(); + throw new RuntimeException("This error should not crash the watcher thread"); + }); + watcher.start(); + awaitWatcherThreadInitialized(); + + LOG.info("Modifying file"); + FileUtils.writeStringToFile(tempFile, "Hello world\n", StandardCharsets.UTF_8, true); + await().atMost(Duration.ofSeconds(2)).untilAsserted(() -> assertEquals(1, callCount.get())); + + // make sure we can still receive the update event, which means the watcher thread is still + // alive + LOG.info("Modifying file again"); + FileUtils.writeStringToFile(tempFile, "Hello world again\n", StandardCharsets.UTF_8, true); + await().atMost(Duration.ofSeconds(2)).untilAsserted(() -> assertEquals(2, callCount.get())); + + // also make sure that the thread is not terminated + assertNotEquals(Thread.State.TERMINATED, watcher.getWatcherThread()); } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java index 4c80d0de0413..447f7af82e75 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.io.util.LRUDictionary; import org.apache.hadoop.hbase.nio.SingleByteBuff; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -56,7 +55,7 @@ public class TestTagCompressionContext { @Test public void testCompressUncompressTags1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext context = new TagCompressionContext(Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); ByteBuffer ib = ByteBuffer.wrap(kv1.getTagsArray()); @@ -83,7 +82,7 @@ public void testCompressUncompressTags1() throws Exception { public void testCompressUncompressTagsWithOffheapKeyValue1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream daos = new ByteBufferWriterDataOutputStream(baos); - TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext context = new TagCompressionContext(Byte.MAX_VALUE); ByteBufferExtendedCell kv1 = (ByteBufferExtendedCell) createOffheapKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); context.compressTags(daos, kv1.getTagsByteBuffer(), kv1.getTagsPosition(), tagsLength1); @@ -107,7 +106,7 @@ public void testCompressUncompressTagsWithOffheapKeyValue1() throws Exception { @Test public void testCompressUncompressTags2() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext context = new TagCompressionContext(Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(1); int tagsLength1 = kv1.getTagsLength(); context.compressTags(baos, kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1); @@ -132,7 +131,7 @@ public void testCompressUncompressTags2() throws Exception { public void testCompressUncompressTagsWithOffheapKeyValue2() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream daos = new ByteBufferWriterDataOutputStream(baos); - TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext context = new TagCompressionContext(Byte.MAX_VALUE); ByteBufferExtendedCell kv1 = (ByteBufferExtendedCell) createOffheapKVWithTags(1); int tagsLength1 = kv1.getTagsLength(); context.compressTags(daos, kv1.getTagsByteBuffer(), kv1.getTagsPosition(), tagsLength1); diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/compress/TestGzipByteBuffDecompressor.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/compress/TestGzipByteBuffDecompressor.java new file mode 100644 index 000000000000..2b86c0ab042d --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/compress/TestGzipByteBuffDecompressor.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.compress; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.zip.GZIPOutputStream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.hbase.nio.MultiByteBuff; +import org.apache.hadoop.hbase.nio.SingleByteBuff; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.util.NativeCodeLoader; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(SmallTests.TAG) +public class TestGzipByteBuffDecompressor { + + // A single gzip member, reused as decompressor input across the tests. + private static final byte[] COMPRESSED_PAYLOAD = gzip("HBase is fun to use and very fast"); + + /** + * GzipByteBuffDecompressor is backed by Hadoop's native zlib binding, so actually decompressing + * anything requires that native library to be loaded on this JVM. + */ + private static void assumeNativeZlibLoaded() { + assumeTrue(NativeCodeLoader.isNativeCodeLoaded(), + "Hadoop's native code is not loaded on this JVM, skipping"); + } + + @Test + public void itReportsCorrectCapabilitiesWithoutNativeZlib() { + ByteBuff emptySingleDirectBuff = new SingleByteBuff(ByteBuffer.allocateDirect(0)); + ByteBuff emptySingleHeapBuff = new SingleByteBuff(ByteBuffer.allocate(0)); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(false)) { + assertFalse(decompressor.canDecompress(emptySingleDirectBuff, emptySingleDirectBuff), + "Without native zlib, direct-to-direct decompression is not available"); + assertFalse(decompressor.canDecompress(emptySingleHeapBuff, emptySingleHeapBuff), + "Heap decompression is not supported; only direct-to-direct"); + } + } + + @Test + public void itReportsCorrectCapabilitiesWithNativeZlib() { + assumeNativeZlibLoaded(); + ByteBuff emptySingleHeapBuff = new SingleByteBuff(ByteBuffer.allocate(0)); + ByteBuff emptyMultiHeapBuff = new MultiByteBuff(ByteBuffer.allocate(0), ByteBuffer.allocate(0)); + ByteBuff emptySingleDirectBuff = new SingleByteBuff(ByteBuffer.allocateDirect(0)); + ByteBuff emptyMultiDirectBuff = + new MultiByteBuff(ByteBuffer.allocateDirect(0), ByteBuffer.allocateDirect(0)); + + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + assertTrue(decompressor.canDecompress(emptySingleDirectBuff, emptySingleDirectBuff)); + // Only direct-to-direct is supported; heap and mixed buffers return false. + assertFalse(decompressor.canDecompress(emptySingleHeapBuff, emptySingleHeapBuff)); + assertFalse(decompressor.canDecompress(emptySingleHeapBuff, emptySingleDirectBuff)); + assertFalse(decompressor.canDecompress(emptySingleDirectBuff, emptySingleHeapBuff)); + assertFalse(decompressor.canDecompress(emptyMultiHeapBuff, emptyMultiHeapBuff)); + assertFalse(decompressor.canDecompress(emptyMultiDirectBuff, emptyMultiDirectBuff)); + assertFalse(decompressor.canDecompress(emptySingleDirectBuff, emptyMultiDirectBuff)); + } + } + + private static ByteBuff directBuffWith(byte[] data) { + ByteBuffer buffer = ByteBuffer.allocateDirect(data.length); + buffer.put(data); + buffer.rewind(); + return new SingleByteBuff(buffer); + } + + private static ByteBuff heapBuffWith(byte[] data) { + ByteBuffer buffer = ByteBuffer.allocate(data.length); + buffer.put(data); + buffer.rewind(); + return new SingleByteBuff(buffer); + } + + private static byte[] gzip(String text) { + ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream out = new GZIPOutputStream(compressed)) { + out.write(Bytes.toBytes(text)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return compressed.toByteArray(); + } + + private static byte[] concat(byte[] first, byte[] second) { + byte[] combined = new byte[first.length + second.length]; + System.arraycopy(first, 0, combined, 0, first.length); + System.arraycopy(second, 0, combined, first.length, second.length); + return combined; + } + + @Test + public void itDecompressesDirectToDirectSuccessfully() throws IOException { + assumeNativeZlibLoaded(); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(COMPRESSED_PAYLOAD); + int decompressedSize = decompressor.decompress(output, input, COMPRESSED_PAYLOAD.length); + assertEquals("HBase is fun to use and very fast", + Bytes.toString(output.toBytes(0, decompressedSize))); + } + } + + @Test + public void itDecompressDirectFailsOnTooShortInput() throws IOException { + assumeNativeZlibLoaded(); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = new SingleByteBuff(ByteBuffer.allocateDirect(10)); + decompressor.decompress(output, input, 10); + fail("Expected an IOException because the input is too short to be a gzip member"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("too short to be a gzip member")); + } + } + + @Test + public void itDecompressDirectFailsOnBadMagicBytes() throws IOException { + assumeNativeZlibLoaded(); + byte[] corrupted = Arrays.copyOf(COMPRESSED_PAYLOAD, COMPRESSED_PAYLOAD.length); + corrupted[0] ^= (byte) 0xff; + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(corrupted); + decompressor.decompress(output, input, corrupted.length); + fail("Expected an IOException because the magic bytes are wrong"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Invalid gzip stream")); + } + } + + @Test + public void itDecompressDirectFailsWhenOutputBufferTooSmall() throws IOException { + assumeNativeZlibLoaded(); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(10)); + ByteBuff input = directBuffWith(COMPRESSED_PAYLOAD); + decompressor.decompress(output, input, COMPRESSED_PAYLOAD.length); + fail("Expected an IOException because the output buffer is too small"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Output buffer is too small")); + } + } + + @Test + public void itDecompressDirectFailsOnCorruptedCrc32() throws IOException { + assumeNativeZlibLoaded(); + byte[] corrupted = Arrays.copyOf(COMPRESSED_PAYLOAD, COMPRESSED_PAYLOAD.length); + // First 4 bytes of the 8-byte trailer are the CRC32, leave ISIZE (the last 4 bytes) alone. + corrupted[corrupted.length - 8] ^= (byte) 0xff; + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(corrupted); + decompressor.decompress(output, input, corrupted.length); + fail("Expected an IOException because the trailer's CRC32 no longer matches"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Invalid gzip stream")); + } + } + + @Test + public void itDecompressDirectFailsOnCorruptedIsize() throws IOException { + assumeNativeZlibLoaded(); + byte[] corrupted = Arrays.copyOf(COMPRESSED_PAYLOAD, COMPRESSED_PAYLOAD.length); + // Last 4 bytes of the 8-byte trailer are the ISIZE. + corrupted[corrupted.length - 4] ^= (byte) 0xff; + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(corrupted); + decompressor.decompress(output, input, corrupted.length); + fail("Expected an IOException because the trailer's ISIZE no longer matches"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Invalid gzip stream")); + } + } + + @Test + public void itDecompressesDirectSuccessfullyOnRepeatedCalls() throws IOException { + assumeNativeZlibLoaded(); + // Mirrors how CodecPool actually uses these: one instance is reused across many blocks, so the + // native decompressor must produce a correct result on every call, not just the first. + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + for (int i = 0; i < 3; i++) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(COMPRESSED_PAYLOAD); + int decompressedSize = decompressor.decompress(output, input, COMPRESSED_PAYLOAD.length); + assertEquals("HBase is fun to use and very fast", + Bytes.toString(output.toBytes(0, decompressedSize))); + } + } + } + + @Test + public void itDecompressDirectIsStillUsableAfterAPreviousCallThrows() throws IOException { + assumeNativeZlibLoaded(); + byte[] corrupted = Arrays.copyOf(COMPRESSED_PAYLOAD, COMPRESSED_PAYLOAD.length); + // First 4 bytes of the 8-byte trailer are the CRC32. + corrupted[corrupted.length - 8] ^= (byte) 0xff; + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff badOutput = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff badInput = directBuffWith(corrupted); + try { + decompressor.decompress(badOutput, badInput, corrupted.length); + fail("Expected an IOException because the trailer's CRC32 no longer matches"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Invalid gzip stream")); + } + + // A prior failure must not leave the shared native decompressor state corrupted for the + // next, valid call. + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(COMPRESSED_PAYLOAD); + int decompressedSize = decompressor.decompress(output, input, COMPRESSED_PAYLOAD.length); + assertEquals("HBase is fun to use and very fast", + Bytes.toString(output.toBytes(0, decompressedSize))); + } + } + + @Test + public void itDecompressesDirectToDirectWithNonZeroBufferPosition() throws IOException { + assumeNativeZlibLoaded(); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuffer rawOutput = ByteBuffer.allocateDirect(128); + rawOutput.position(32); + + ByteBuffer rawInput = ByteBuffer.allocateDirect(16 + COMPRESSED_PAYLOAD.length); + for (int i = 0; i < 16; i++) { + rawInput.put((byte) 0); + } + rawInput.put(COMPRESSED_PAYLOAD); + rawInput.position(16); + + ByteBuff output = new SingleByteBuff(rawOutput); + ByteBuff input = new SingleByteBuff(rawInput); + int decompressedSize = decompressor.decompress(output, input, COMPRESSED_PAYLOAD.length); + + byte[] result = new byte[decompressedSize]; + rawOutput.position(32); + rawOutput.get(result); + assertEquals("HBase is fun to use and very fast", Bytes.toString(result)); + } + } + + @Test + public void itDecompressDirectFailsOnTruncatedGzipStream() throws IOException { + assumeNativeZlibLoaded(); + byte[] truncated = Arrays.copyOf(COMPRESSED_PAYLOAD, COMPRESSED_PAYLOAD.length - 4); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(truncated); + decompressor.decompress(output, input, truncated.length); + fail("Expected an IOException because the gzip stream is truncated"); + } catch (IOException e) { + // Expected: the decompressor must not report finished() on an incomplete stream + } + } + + @Test + public void itDecompressesOnlyTheDelimitedMemberFromAMultiMemberPayload() throws IOException { + assumeNativeZlibLoaded(); + // Two distinct members concatenated: we must decode only the one delimited by inputLen. + byte[] firstMember = gzip("first member"); + byte[] secondMember = gzip("second member"); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(concat(firstMember, secondMember)); + int decompressedSize = decompressor.decompress(output, input, firstMember.length); + assertEquals("first member", Bytes.toString(output.toBytes(0, decompressedSize))); + } + } + + /** + * This is the exact gate {@code HFileBlockDefaultDecodingContext#canDecompressViaByteBuff} relies + * on to decide between ByteBuff decompression and the stream path, driven end-to-end from the + * {@code GzipHFileDecompressionContext#ALLOW_BYTE_BUFF_DECOMPRESSION_KEY} config flag. + */ + @Test + public void itReinitControlsByteBuffDecompressionViaConfigFlag() { + assumeNativeZlibLoaded(); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(COMPRESSED_PAYLOAD); + + Configuration conf = new Configuration(false); + conf.setBoolean(GzipHFileDecompressionContext.ALLOW_BYTE_BUFF_DECOMPRESSION_KEY, false); + decompressor.reinit(GzipHFileDecompressionContext.fromConfiguration(conf)); + assertFalse(decompressor.canDecompress(output, input), + "Block reader must fall back to stream decompression when the config flag " + + "disables ByteBuff decompression"); + + conf.setBoolean(GzipHFileDecompressionContext.ALLOW_BYTE_BUFF_DECOMPRESSION_KEY, true); + decompressor.reinit(GzipHFileDecompressionContext.fromConfiguration(conf)); + assertTrue(decompressor.canDecompress(output, input), + "Block reader must use ByteBuff decompression when the config flag is enabled"); + + // The default, with no config value set, must also allow ByteBuff decompression. + decompressor + .reinit(GzipHFileDecompressionContext.fromConfiguration(new Configuration(false))); + assertTrue(decompressor.canDecompress(output, input)); + } + } + + @Test + public void itReinitWithNullContextIsNoOp() { + assumeNativeZlibLoaded(); + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(true)) { + ByteBuff output = new SingleByteBuff(ByteBuffer.allocateDirect(64)); + ByteBuff input = directBuffWith(COMPRESSED_PAYLOAD); + + Configuration conf = new Configuration(false); + conf.setBoolean(GzipHFileDecompressionContext.ALLOW_BYTE_BUFF_DECOMPRESSION_KEY, false); + decompressor.reinit(GzipHFileDecompressionContext.fromConfiguration(conf)); + assertFalse(decompressor.canDecompress(output, input)); + + decompressor.reinit(null); + assertFalse(decompressor.canDecompress(output, input), + "reinit(null) must not reset allowByteBuffDecompression back to the default"); + } + } + + @Test + public void itReinitFailsOnWrongContextType() { + Compression.HFileDecompressionContext wrongContext = + new Compression.HFileDecompressionContext() { + @Override + public void close() { + } + + @Override + public long heapSize() { + return 0; + } + }; + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(false)) { + decompressor.reinit(wrongContext); + fail("Expected an IllegalArgumentException because the context was not a " + + "GzipHFileDecompressionContext"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("GzipHFileDecompressionContext")); + } + } + + @Test + public void itDecompressThrowsWhenPassedAMultiByteBuff() throws IOException { + try (GzipByteBuffDecompressor decompressor = new GzipByteBuffDecompressor(false)) { + ByteBuff multiOutput = new MultiByteBuff(ByteBuffer.allocate(64), ByteBuffer.allocate(64)); + ByteBuff input = heapBuffWith(COMPRESSED_PAYLOAD); + decompressor.decompress(multiOutput, input, COMPRESSED_PAYLOAD.length); + fail("Expected an IllegalStateException when output is a MultiByteBuff"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("not a SingleByteBuff")); + } + } + +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestUndoableLRUDictionary.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestUndoableLRUDictionary.java new file mode 100644 index 000000000000..4f6bef827a6b --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestUndoableLRUDictionary.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MiscTests.class, SmallTests.class }) +public class TestUndoableLRUDictionary { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestUndoableLRUDictionary.class); + + private UndoableLRUDictionary dict; + private static final int CAPACITY = 4; + + @Before + public void setUp() { + dict = new UndoableLRUDictionary(); + dict.init(CAPACITY); + } + + private static byte[] entry(String s) { + return Bytes.toBytes(s); + } + + private short add(String s) { + byte[] data = entry(s); + return dict.addEntry(data, 0, data.length); + } + + private void assertNoEntry(short idx) { + assertThrows(IndexOutOfBoundsException.class, () -> dict.getEntry(idx)); + } + + @Test + public void itPreservesAdditionsOnCommit() { + add("a"); + add("b"); + + dict.checkpoint(); + add("c"); + dict.commit(); + + assertArrayEquals(entry("a"), dict.getEntry((short) 0)); + assertArrayEquals(entry("b"), dict.getEntry((short) 1)); + assertArrayEquals(entry("c"), dict.getEntry((short) 2)); + } + + @Test + public void itRevertsAdditionsOnRollback() { + short idxA = add("a"); + short idxB = add("b"); + + dict.checkpoint(); + add("c"); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertNoEntry((short) 2); + } + + @Test + public void itIsNoOpWhenRollingBackWithoutCheckpoint() { + short idxA = add("a"); + short idxB = add("b"); + + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + } + + @Test + public void itRestoresEvictedEntryOnRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + short idxE = add("e"); + assertEquals(idxA, idxE); + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesAddGetAddPatternAtCapacityThenRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + + short idxE = add("e"); + byte[] gotC = dict.getEntry(idxC); + assertArrayEquals(entry("c"), gotC); + short idxF = add("f"); + + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + assertArrayEquals(entry("f"), dict.getEntry(idxF)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesAddGetAddPatternAtCapacityThenCommit() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + short idxE = add("e"); + dict.getEntry(idxC); + short idxF = add("f"); + dict.commit(); + + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + assertArrayEquals(entry("f"), dict.getEntry(idxF)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itRestoresLruOrderAfterGetEntryOnRollback() { + short idxA = add("a"); + add("b"); + add("c"); + + dict.checkpoint(); + dict.getEntry(idxA); + add("d"); + dict.rollback(); + + assertNoEntry((short) 3); + add("d"); + short idxE = add("e"); + assertEquals(idxA, idxE); + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + } + + @Test + public void itSupportsMultipleCheckpointRollbackCycles() { + short idxA = add("a"); + short idxB = add("b"); + + dict.checkpoint(); + add("c"); + dict.rollback(); + + assertNoEntry((short) 2); + + dict.checkpoint(); + add("d"); + dict.rollback(); + + assertNoEntry((short) 2); + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + } + + @Test + public void itSupportsMultipleCheckpointCommitCycles() { + dict.checkpoint(); + short idxA = add("a"); + dict.commit(); + + dict.checkpoint(); + short idxB = add("b"); + dict.commit(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + } + + @Test + public void itPreservesEvictionsOnCommit() { + short idxA = add("a"); + add("b"); + add("c"); + add("d"); + + dict.checkpoint(); + short idxE = add("e"); + assertEquals(idxA, idxE); + dict.commit(); + + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + } + + @Test + public void itHandlesMultipleEvictionsDuringTrackingThenRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + add("e"); + add("f"); + add("g"); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesEvictionThenGetEvictedIndexThenRollback() { + short idxA = add("a"); + add("b"); + add("c"); + add("d"); + + dict.checkpoint(); + short idxE = add("e"); + assertEquals(idxA, idxE); + byte[] gotE = dict.getEntry(idxE); + assertArrayEquals(entry("e"), gotE); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + } + + @Test + public void itHandlesRollbackAfterAddBelowCapacity() { + dict.checkpoint(); + add("a"); + add("b"); + dict.rollback(); + + assertNoEntry((short) 0); + } + + @Test + public void itHandlesCommitThenRollbackSequence() { + short idxA = add("a"); + short idxB = add("b"); + + dict.checkpoint(); + add("c"); + dict.commit(); + + assertArrayEquals(entry("c"), dict.getEntry((short) 2)); + + dict.checkpoint(); + add("d"); + dict.rollback(); + + assertNoEntry((short) 3); + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + } + + @Test + public void itHandlesGetEntryForEverySlotDuringTrackingThenRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + dict.getEntry(idxA); + dict.getEntry(idxB); + dict.getEntry(idxC); + dict.getEntry(idxD); + add("e"); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesAtCapacityEvictAllThenRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + add("e"); + add("f"); + add("g"); + add("h"); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesRollbackWhenEvictedValueReaddedToAnotherSlot() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + short idxE = add("e"); + assertEquals(idxA, idxE); + short idxA2 = add("a"); + assertEquals(idxB, idxA2); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ByteBufferUtilsTestBase.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ByteBufferUtilsTestBase.java new file mode 100644 index 000000000000..194915475775 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ByteBufferUtilsTestBase.java @@ -0,0 +1,600 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.io.WritableUtils; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ByteBufferUtilsTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(ByteBufferUtilsTestBase.class); + + private static int MAX_VLONG_LENGTH = 9; + private static Collection testNumbers; + + private byte[] array; + + @BeforeAll + public static void setUpBeforeAll() { + SortedSet a = new TreeSet<>(); + for (int i = 0; i <= 63; ++i) { + long v = -1L << i; + assertTrue(v < 0); + addNumber(a, v); + v = (1L << i) - 1; + assertTrue(v >= 0); + addNumber(a, v); + } + + testNumbers = Collections.unmodifiableSet(a); + LOG.info("Testing variable-length long serialization using: {} (count: {})", testNumbers, + testNumbers.size()); + assertEquals(1753, testNumbers.size()); + assertEquals(Long.MIN_VALUE, a.first().longValue()); + assertEquals(Long.MAX_VALUE, a.last().longValue()); + } + + /** + * Create an array with sample data. + */ + @BeforeEach + public void setUp() { + array = new byte[8]; + for (int i = 0; i < array.length; ++i) { + array[i] = (byte) ('a' + i); + } + } + + private static void addNumber(Set a, long l) { + if (l != Long.MIN_VALUE) { + a.add(l - 1); + } + a.add(l); + if (l != Long.MAX_VALUE) { + a.add(l + 1); + } + for (long divisor = 3; divisor <= 10; ++divisor) { + for (long delta = -1; delta <= 1; ++delta) { + a.add(l / divisor + delta); + } + } + } + + @Test + public void testReadWriteVLong() { + for (long l : testNumbers) { + ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(b)); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); + } + } + + @Test + public void testReadWriteConsecutiveVLong() { + for (long l : testNumbers) { + ByteBuffer b = ByteBuffer.allocate(2 * MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + ByteBufferUtils.writeVLong(b, l - 4); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(b)); + assertEquals(l - 4, ByteBufferUtils.readVLong(b)); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); + assertEquals(l - 4, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); + } + } + + @Test + public void testConsistencyWithHadoopVLong() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + for (long l : testNumbers) { + baos.reset(); + ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + String bufStr = Bytes.toStringBinary(b.array(), b.arrayOffset(), b.position()); + WritableUtils.writeVLong(dos, l); + String baosStr = Bytes.toStringBinary(baos.toByteArray()); + assertEquals(baosStr, bufStr); + } + } + + /** + * Test copying to stream from buffer. + */ + @Test + public void testMoveBufferToStream() throws IOException { + final int arrayOffset = 7; + final int initialPosition = 10; + final int endPadding = 5; + byte[] arrayWrapper = new byte[arrayOffset + initialPosition + array.length + endPadding]; + System.arraycopy(array, 0, arrayWrapper, arrayOffset + initialPosition, array.length); + ByteBuffer buffer = + ByteBuffer.wrap(arrayWrapper, arrayOffset, initialPosition + array.length).slice(); + assertEquals(initialPosition + array.length, buffer.limit()); + assertEquals(0, buffer.position()); + buffer.position(initialPosition); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteBufferUtils.moveBufferToStream(bos, buffer, array.length); + assertArrayEquals(array, bos.toByteArray()); + assertEquals(initialPosition + array.length, buffer.position()); + } + + /** + * Test copying to stream from buffer with offset. + * @throws IOException On test failure. + */ + @Test + public void testCopyToStreamWithOffset() throws IOException { + ByteBuffer buffer = ByteBuffer.wrap(array); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + + ByteBufferUtils.copyBufferToStream(bos, buffer, array.length / 2, array.length / 2); + + byte[] returnedArray = bos.toByteArray(); + for (int i = 0; i < array.length / 2; ++i) { + int pos = array.length / 2 + i; + assertEquals(returnedArray[i], array[pos]); + } + } + + /** + * Test copying data from stream. + * @throws IOException On test failure. + */ + @Test + public void testCopyFromStream() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(array.length); + ByteArrayInputStream bis = new ByteArrayInputStream(array); + DataInputStream dis = new DataInputStream(bis); + + ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length / 2); + ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length - array.length / 2); + for (int i = 0; i < array.length; ++i) { + assertEquals(array[i], buffer.get(i)); + } + } + + /** + * Test copying from buffer. + */ + @Test + public void testCopyFromBuffer() { + ByteBuffer srcBuffer = ByteBuffer.allocate(array.length); + ByteBuffer dstBuffer = ByteBuffer.allocate(array.length); + srcBuffer.put(array); + + ByteBufferUtils.copyFromBufferToBuffer(srcBuffer, dstBuffer, array.length / 2, + array.length / 4); + for (int i = 0; i < array.length / 4; ++i) { + assertEquals(srcBuffer.get(i + array.length / 2), dstBuffer.get(i)); + } + } + + /** + * Test 7-bit encoding of integers. + * @throws IOException On test failure. + */ + @Test + public void testCompressedInt() throws IOException { + testCompressedInt(0); + testCompressedInt(Integer.MAX_VALUE); + testCompressedInt(Integer.MIN_VALUE); + + for (int i = 0; i < 3; i++) { + testCompressedInt((128 << i) - 1); + } + + for (int i = 0; i < 3; i++) { + testCompressedInt((128 << i)); + } + } + + /** + * Test how much bytes we need to store integer. + */ + @Test + public void testIntFitsIn() { + assertEquals(1, ByteBufferUtils.intFitsIn(0)); + assertEquals(1, ByteBufferUtils.intFitsIn(1)); + assertEquals(2, ByteBufferUtils.intFitsIn(1 << 8)); + assertEquals(3, ByteBufferUtils.intFitsIn(1 << 16)); + assertEquals(4, ByteBufferUtils.intFitsIn(-1)); + assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MAX_VALUE)); + assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MIN_VALUE)); + } + + /** + * Test how much bytes we need to store long. + */ + @Test + public void testLongFitsIn() { + assertEquals(1, ByteBufferUtils.longFitsIn(0)); + assertEquals(1, ByteBufferUtils.longFitsIn(1)); + assertEquals(3, ByteBufferUtils.longFitsIn(1L << 16)); + assertEquals(5, ByteBufferUtils.longFitsIn(1L << 32)); + assertEquals(8, ByteBufferUtils.longFitsIn(-1)); + assertEquals(8, ByteBufferUtils.longFitsIn(Long.MIN_VALUE)); + assertEquals(8, ByteBufferUtils.longFitsIn(Long.MAX_VALUE)); + } + + /** + * Test if we are comparing equal bytes. + */ + @Test + public void testArePartEqual() { + byte[] array = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4 }; + ByteBuffer buffer = ByteBuffer.wrap(array); + assertTrue(ByteBufferUtils.arePartsEqual(buffer, 0, 4, 5, 4)); + assertTrue(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 2)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 3)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 3, 6, 2)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 0, 3, 6, 3)); + } + + /** + * Test serializing int to bytes + */ + @Test + public void testPutInt() { + testPutInt(0); + testPutInt(Integer.MAX_VALUE); + + for (int i = 0; i < 3; i++) { + testPutInt((128 << i) - 1); + } + + for (int i = 0; i < 3; i++) { + testPutInt((128 << i)); + } + } + + @Test + public void testToBytes() { + ByteBuffer buffer = ByteBuffer.allocate(5); + buffer.put(new byte[] { 0, 1, 2, 3, 4 }); + assertEquals(5, buffer.position()); + assertEquals(5, buffer.limit()); + byte[] copy = ByteBufferUtils.toBytes(buffer, 2); + assertArrayEquals(new byte[] { 2, 3, 4 }, copy); + assertEquals(5, buffer.position()); + assertEquals(5, buffer.limit()); + } + + @Test + public void testToPrimitiveTypes() { + ByteBuffer buffer = ByteBuffer.allocate(15); + long l = 988L; + int i = 135; + short s = 7; + buffer.putLong(l); + buffer.putShort(s); + buffer.putInt(i); + assertEquals(l, ByteBufferUtils.toLong(buffer, 0)); + assertEquals(s, ByteBufferUtils.toShort(buffer, 8)); + assertEquals(i, ByteBufferUtils.toInt(buffer, 10)); + } + + @Test + public void testCopyFromArrayToBuffer() { + byte[] b = new byte[15]; + b[0] = -1; + long l = 988L; + int i = 135; + short s = 7; + Bytes.putLong(b, 1, l); + Bytes.putShort(b, 9, s); + Bytes.putInt(b, 11, i); + ByteBuffer buffer = ByteBuffer.allocate(14); + ByteBufferUtils.copyFromArrayToBuffer(buffer, b, 1, 14); + buffer.rewind(); + assertEquals(l, buffer.getLong()); + assertEquals(s, buffer.getShort()); + assertEquals(i, buffer.getInt()); + } + + private void testCopyFromSrcToDestWithThreads(Object input, Object output, List lengthes, + List offsets) throws InterruptedException { + assertTrue((input instanceof ByteBuffer) || (input instanceof byte[])); + assertTrue((output instanceof ByteBuffer) || (output instanceof byte[])); + assertEquals(lengthes.size(), offsets.size()); + + final int threads = lengthes.size(); + CountDownLatch latch = new CountDownLatch(1); + List exes = new ArrayList<>(threads); + int oldInputPos = (input instanceof ByteBuffer) ? ((ByteBuffer) input).position() : 0; + int oldOutputPos = (output instanceof ByteBuffer) ? ((ByteBuffer) output).position() : 0; + for (int i = 0; i != threads; ++i) { + int offset = offsets.get(i); + int length = lengthes.get(i); + exes.add(() -> { + try { + latch.await(); + if (input instanceof ByteBuffer && output instanceof byte[]) { + ByteBufferUtils.copyFromBufferToArray((byte[]) output, (ByteBuffer) input, offset, + offset, length); + } + if (input instanceof byte[] && output instanceof ByteBuffer) { + ByteBufferUtils.copyFromArrayToBuffer((ByteBuffer) output, offset, (byte[]) input, + offset, length); + } + if (input instanceof ByteBuffer && output instanceof ByteBuffer) { + ByteBufferUtils.copyFromBufferToBuffer((ByteBuffer) input, (ByteBuffer) output, offset, + offset, length); + } + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + }); + } + ExecutorService service = Executors.newFixedThreadPool(threads); + exes.forEach(service::execute); + latch.countDown(); + service.shutdown(); + assertTrue(service.awaitTermination(5, TimeUnit.SECONDS)); + if (input instanceof ByteBuffer) { + assertEquals(oldInputPos, ((ByteBuffer) input).position()); + } + if (output instanceof ByteBuffer) { + assertEquals(oldOutputPos, ((ByteBuffer) output).position()); + } + String inputString = (input instanceof ByteBuffer) + ? Bytes.toString(Bytes.toBytes((ByteBuffer) input)) + : Bytes.toString((byte[]) input); + String outputString = (output instanceof ByteBuffer) + ? Bytes.toString(Bytes.toBytes((ByteBuffer) output)) + : Bytes.toString((byte[]) output); + assertEquals(inputString, outputString); + } + + @Test + public void testCopyFromSrcToDestWithThreads() throws InterruptedException { + List words = + Arrays.asList(Bytes.toBytes("with"), Bytes.toBytes("great"), Bytes.toBytes("power"), + Bytes.toBytes("comes"), Bytes.toBytes("great"), Bytes.toBytes("responsibility")); + List lengthes = words.stream().map(v -> v.length).collect(Collectors.toList()); + List offsets = new ArrayList<>(words.size()); + for (int i = 0; i != words.size(); ++i) { + offsets.add(words.subList(0, i).stream().mapToInt(v -> v.length).sum()); + } + + int totalSize = words.stream().mapToInt(v -> v.length).sum(); + byte[] fullContent = new byte[totalSize]; + int offset = 0; + for (byte[] w : words) { + offset = Bytes.putBytes(fullContent, offset, w, 0, w.length); + } + + // test copyFromBufferToArray + for (ByteBuffer input : Arrays.asList(ByteBuffer.allocateDirect(totalSize), + ByteBuffer.allocate(totalSize))) { + words.forEach(input::put); + byte[] output = new byte[totalSize]; + testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); + } + + // test copyFromArrayToBuffer + for (ByteBuffer output : Arrays.asList(ByteBuffer.allocateDirect(totalSize), + ByteBuffer.allocate(totalSize))) { + byte[] input = fullContent; + testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); + } + + // test copyFromBufferToBuffer + for (ByteBuffer input : Arrays.asList(ByteBuffer.allocateDirect(totalSize), + ByteBuffer.allocate(totalSize))) { + words.forEach(input::put); + for (ByteBuffer output : Arrays.asList(ByteBuffer.allocateDirect(totalSize), + ByteBuffer.allocate(totalSize))) { + testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); + } + } + } + + @Test + public void testCopyFromBufferToArray() { + ByteBuffer buffer = ByteBuffer.allocate(15); + buffer.put((byte) -1); + long l = 988L; + int i = 135; + short s = 7; + buffer.putShort(s); + buffer.putInt(i); + buffer.putLong(l); + byte[] b = new byte[15]; + ByteBufferUtils.copyFromBufferToArray(b, buffer, 1, 1, 14); + assertEquals(s, Bytes.toShort(b, 1)); + assertEquals(i, Bytes.toInt(b, 3)); + assertEquals(l, Bytes.toLong(b, 7)); + } + + @Test + public void testRelativeCopyFromBuffertoBuffer() { + ByteBuffer bb1 = ByteBuffer.allocate(135); + ByteBuffer bb2 = ByteBuffer.allocate(135); + fillBB(bb1, (byte) 5); + ByteBufferUtils.copyFromBufferToBuffer(bb1, bb2); + assertTrue(bb1.position() == bb2.position()); + assertTrue(bb1.limit() == bb2.limit()); + bb1 = ByteBuffer.allocateDirect(135); + bb2 = ByteBuffer.allocateDirect(135); + fillBB(bb1, (byte) 5); + ByteBufferUtils.copyFromBufferToBuffer(bb1, bb2); + assertTrue(bb1.position() == bb2.position()); + assertTrue(bb1.limit() == bb2.limit()); + } + + @Test + public void testCompareTo() { + ByteBuffer bb1 = ByteBuffer.allocate(135); + ByteBuffer bb2 = ByteBuffer.allocate(135); + byte[] b = new byte[71]; + fillBB(bb1, (byte) 5); + fillBB(bb2, (byte) 5); + fillArray(b, (byte) 5); + assertEquals(0, ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); + assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), b, 0, b.length) > 0); + bb2.put(134, (byte) 6); + assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()) < 0); + bb2.put(6, (byte) 4); + assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()) > 0); + // Assert reverse comparing BB and bytearray works. + ByteBuffer bb3 = ByteBuffer.allocate(135); + fillBB(bb3, (byte) 0); + byte[] b3 = new byte[135]; + fillArray(b3, (byte) 1); + int result = ByteBufferUtils.compareTo(b3, 0, b3.length, bb3, 0, bb3.remaining()); + assertTrue(result > 0); + result = ByteBufferUtils.compareTo(bb3, 0, bb3.remaining(), b3, 0, b3.length); + assertTrue(result < 0); + byte[] b4 = Bytes.toBytes("123"); + ByteBuffer bb4 = ByteBuffer.allocate(10 + b4.length); + for (int i = 10; i < bb4.capacity(); ++i) { + bb4.put(i, b4[i - 10]); + } + result = ByteBufferUtils.compareTo(b4, 0, b4.length, bb4, 10, b4.length); + assertEquals(0, result); + } + + @Test + public void testEquals() { + byte[] a = Bytes.toBytes("http://A"); + ByteBuffer bb = ByteBuffer.wrap(a); + + assertTrue(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, + HConstants.EMPTY_BYTE_BUFFER, 0, 0)); + + assertFalse(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, bb, 0, a.length)); + + assertFalse(ByteBufferUtils.equals(bb, 0, 0, HConstants.EMPTY_BYTE_BUFFER, 0, a.length)); + + assertTrue(ByteBufferUtils.equals(bb, 0, a.length, bb, 0, a.length)); + + assertTrue(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, + HConstants.EMPTY_BYTE_ARRAY, 0, 0)); + + assertFalse(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, a, 0, a.length)); + + assertFalse(ByteBufferUtils.equals(bb, 0, a.length, HConstants.EMPTY_BYTE_ARRAY, 0, 0)); + + assertTrue(ByteBufferUtils.equals(bb, 0, a.length, a, 0, a.length)); + } + + @Test + public void testFindCommonPrefix() { + ByteBuffer bb1 = ByteBuffer.allocate(135); + ByteBuffer bb2 = ByteBuffer.allocate(135); + ByteBuffer bb3 = ByteBuffer.allocateDirect(135); + byte[] b = new byte[71]; + + fillBB(bb1, (byte) 5); + fillBB(bb2, (byte) 5); + fillBB(bb3, (byte) 5); + fillArray(b, (byte) 5); + + assertEquals(135, + ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); + assertEquals(71, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length)); + assertEquals(135, + ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb3, 0, bb3.remaining())); + assertEquals(71, ByteBufferUtils.findCommonPrefix(bb3, 0, bb3.remaining(), b, 0, b.length)); + + b[13] = 9; + assertEquals(13, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length)); + + bb2.put(134, (byte) 6); + assertEquals(134, + ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); + + bb2.put(6, (byte) 4); + assertEquals(6, + ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); + } + + // Below are utility methods invoked from test methods + private static void testCompressedInt(int value) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteBufferUtils.putCompressedInt(bos, value); + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + int parsedValue = ByteBufferUtils.readCompressedInt(bis); + assertEquals(value, parsedValue); + } + + private static void testPutInt(int value) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + ByteBufferUtils.putInt(baos, value); + } catch (IOException e) { + throw new RuntimeException("Bug in putIn()", e); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + try { + assertEquals(dis.readInt(), value); + } catch (IOException e) { + throw new RuntimeException("Bug in test!", e); + } + } + + private static void fillBB(ByteBuffer bb, byte b) { + for (int i = bb.position(); i < bb.limit(); i++) { + bb.put(i, b); + } + } + + private static void fillArray(byte[] bb, byte b) { + for (int i = 0; i < bb.length; i++) { + bb[i] = b; + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/BytesTestBase.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/BytesTestBase.java new file mode 100644 index 000000000000..96df8bc39396 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/BytesTestBase.java @@ -0,0 +1,578 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.hadoop.io.WritableUtils; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BytesTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(BytesTestBase.class); + + @Test + public void testShort() throws Exception { + for (short n : Arrays.asList(Short.MIN_VALUE, (short) -100, (short) -1, (short) 0, (short) 1, + (short) 300, Short.MAX_VALUE)) { + byte[] bytes = Bytes.toBytes(n); + assertEquals(Bytes.toShort(bytes, 0, bytes.length), n); + } + } + + @Test + public void testNullHashCode() { + byte[] b = null; + Exception ee = null; + try { + Bytes.hashCode(b); + } catch (Exception e) { + ee = e; + } + assertNotNull(ee); + } + + @Test + public void testAdd() { + byte[] a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + byte[] b = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + byte[] c = { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }; + byte[] result1 = Bytes.add(a, b, c); + byte[] result2 = Bytes.add(new byte[][] { a, b, c }); + assertEquals(0, Bytes.compareTo(result1, result2)); + } + + @Test + public void testSplit() { + byte[] lowest = Bytes.toBytes("AAA"); + byte[] middle = Bytes.toBytes("CCC"); + byte[] highest = Bytes.toBytes("EEE"); + byte[][] parts = Bytes.split(lowest, highest, 1); + for (byte[] bytes : parts) { + LOG.info(Bytes.toString(bytes)); + } + assertEquals(3, parts.length); + assertTrue(Bytes.equals(parts[1], middle)); + // Now divide into three parts. Change highest so split is even. + highest = Bytes.toBytes("DDD"); + parts = Bytes.split(lowest, highest, 2); + for (byte[] part : parts) { + LOG.info(Bytes.toString(part)); + } + assertEquals(4, parts.length); + // Assert that 3rd part is 'CCC'. + assertTrue(Bytes.equals(parts[2], middle)); + } + + @Test + public void testSplit2() { + // More split tests. + byte[] lowest = Bytes.toBytes("http://A"); + byte[] highest = Bytes.toBytes("http://z"); + byte[] middle = Bytes.toBytes("http://]"); + byte[][] parts = Bytes.split(lowest, highest, 1); + for (byte[] part : parts) { + LOG.info(Bytes.toString(part)); + } + assertEquals(3, parts.length); + assertTrue(Bytes.equals(parts[1], middle)); + } + + @Test + public void testSplit3() { + // Test invalid split cases + byte[] low = { 1, 1, 1 }; + byte[] high = { 1, 1, 3 }; + + // If swapped, should throw IAE + try { + Bytes.split(high, low, 1); + fail("Should not be able to split if low > high"); + } catch (IllegalArgumentException iae) { + // Correct + } + + // Single split should work + byte[][] parts = Bytes.split(low, high, 1); + for (int i = 0; i < parts.length; i++) { + LOG.info("" + i + " -> " + Bytes.toStringBinary(parts[i])); + } + assertEquals(3, parts.length, "Returned split should have 3 parts but has " + parts.length); + + // If split more than once, use additional byte to split + parts = Bytes.split(low, high, 2); + assertNotNull(parts, "Split with an additional byte"); + assertEquals(parts.length, low.length + 1); + + // Split 0 times should throw IAE + try { + Bytes.split(low, high, 0); + fail("Should not be able to split 0 times"); + } catch (IllegalArgumentException iae) { + // Correct + } + } + + @Test + public void testToInt() { + int[] ints = { -1, 123, Integer.MIN_VALUE, Integer.MAX_VALUE }; + for (int anInt : ints) { + byte[] b = Bytes.toBytes(anInt); + assertEquals(anInt, Bytes.toInt(b)); + byte[] b2 = bytesWithOffset(b); + assertEquals(anInt, Bytes.toInt(b2, 1)); + assertEquals(anInt, Bytes.toInt(b2, 1, Bytes.SIZEOF_INT)); + } + } + + @Test + public void testToLong() { + long[] longs = { -1L, 123L, Long.MIN_VALUE, Long.MAX_VALUE }; + for (long aLong : longs) { + byte[] b = Bytes.toBytes(aLong); + assertEquals(aLong, Bytes.toLong(b)); + byte[] b2 = bytesWithOffset(b); + assertEquals(aLong, Bytes.toLong(b2, 1)); + assertEquals(aLong, Bytes.toLong(b2, 1, Bytes.SIZEOF_LONG)); + } + } + + @Test + public void testToFloat() { + float[] floats = { -1f, 123.123f, Float.MAX_VALUE }; + for (float aFloat : floats) { + byte[] b = Bytes.toBytes(aFloat); + assertEquals(aFloat, Bytes.toFloat(b), 0.0f); + byte[] b2 = bytesWithOffset(b); + assertEquals(aFloat, Bytes.toFloat(b2, 1), 0.0f); + } + } + + @Test + public void testToDouble() { + double[] doubles = { Double.MIN_VALUE, Double.MAX_VALUE }; + for (double aDouble : doubles) { + byte[] b = Bytes.toBytes(aDouble); + assertEquals(aDouble, Bytes.toDouble(b), 0.0); + byte[] b2 = bytesWithOffset(b); + assertEquals(aDouble, Bytes.toDouble(b2, 1), 0.0); + } + } + + @Test + public void testToBigDecimal() { + BigDecimal[] decimals = + { new BigDecimal("-1"), new BigDecimal("123.123"), new BigDecimal("123123123123") }; + for (BigDecimal decimal : decimals) { + byte[] b = Bytes.toBytes(decimal); + assertEquals(decimal, Bytes.toBigDecimal(b)); + byte[] b2 = bytesWithOffset(b); + assertEquals(decimal, Bytes.toBigDecimal(b2, 1, b.length)); + } + } + + private byte[] bytesWithOffset(byte[] src) { + // add one byte in front to test offset + byte[] result = new byte[src.length + 1]; + result[0] = (byte) 0xAA; + System.arraycopy(src, 0, result, 1, src.length); + return result; + } + + @Test + public void testToBytesForByteBuffer() { + byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + ByteBuffer target = ByteBuffer.wrap(array); + target.position(2); + target.limit(7); + + byte[] actual = Bytes.toBytes(target); + byte[] expected = { 0, 1, 2, 3, 4, 5, 6 }; + assertArrayEquals(expected, actual); + assertEquals(2, target.position()); + assertEquals(7, target.limit()); + + ByteBuffer target2 = target.slice(); + assertEquals(0, target2.position()); + assertEquals(5, target2.limit()); + + byte[] actual2 = Bytes.toBytes(target2); + byte[] expected2 = { 2, 3, 4, 5, 6 }; + assertArrayEquals(expected2, actual2); + assertEquals(0, target2.position()); + assertEquals(5, target2.limit()); + } + + @Test + public void testGetBytesForByteBuffer() { + byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + ByteBuffer target = ByteBuffer.wrap(array); + target.position(2); + target.limit(7); + + byte[] actual = Bytes.getBytes(target); + byte[] expected = { 2, 3, 4, 5, 6 }; + assertArrayEquals(expected, actual); + assertEquals(2, target.position()); + assertEquals(7, target.limit()); + } + + @Test + public void testReadAsVLong() throws Exception { + long[] longs = { -1L, 123L, Long.MIN_VALUE, Long.MAX_VALUE }; + for (long aLong : longs) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(baos); + WritableUtils.writeVLong(output, aLong); + byte[] long_bytes_no_offset = baos.toByteArray(); + assertEquals(aLong, Bytes.readAsVLong(long_bytes_no_offset, 0)); + byte[] long_bytes_with_offset = bytesWithOffset(long_bytes_no_offset); + assertEquals(aLong, Bytes.readAsVLong(long_bytes_with_offset, 1)); + } + } + + @Test + public void testToStringBinaryForBytes() { + byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; + String actual = Bytes.toStringBinary(array); + String expected = "09azAZ@\\x01"; + assertEquals(expected, actual); + + String actual2 = Bytes.toStringBinary(array, 2, 3); + String expected2 = "azA"; + assertEquals(expected2, actual2); + } + + @Test + public void testToStringBinaryForArrayBasedByteBuffer() { + byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; + ByteBuffer target = ByteBuffer.wrap(array); + String actual = Bytes.toStringBinary(target); + String expected = "09azAZ@\\x01"; + assertEquals(expected, actual); + } + + @Test + public void testToStringBinaryForReadOnlyByteBuffer() { + byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; + ByteBuffer target = ByteBuffer.wrap(array).asReadOnlyBuffer(); + String actual = Bytes.toStringBinary(target); + String expected = "09azAZ@\\x01"; + assertEquals(expected, actual); + } + + @Test + public void testBinarySearch() { + byte[][] arr = { { 1 }, { 3 }, { 5 }, { 7 }, { 9 }, { 11 }, { 13 }, { 15 }, }; + byte[] key1 = { 3, 1 }; + byte[] key2 = { 4, 9 }; + byte[] key2_2 = { 4 }; + byte[] key3 = { 5, 11 }; + byte[] key4 = { 0 }; + byte[] key5 = { 2 }; + + assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1)); + assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1)); + assertEquals(-(2 + 1), Arrays.binarySearch(arr, key2_2, Bytes.BYTES_COMPARATOR)); + assertEquals(-(2 + 1), Bytes.binarySearch(arr, key2, 0, 1)); + assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1)); + assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1)); + assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1)); + assertEquals(-1, Bytes.binarySearch(arr, key4, 0, 1)); + assertEquals(-2, Bytes.binarySearch(arr, key5, 0, 1)); + + // Search for values to the left and to the right of each item in the array. + for (int i = 0; i < arr.length; ++i) { + assertEquals(-(i + 1), Bytes.binarySearch(arr, new byte[] { (byte) (arr[i][0] - 1) }, 0, 1)); + assertEquals(-(i + 2), Bytes.binarySearch(arr, new byte[] { (byte) (arr[i][0] + 1) }, 0, 1)); + } + } + + @Test + public void testToStringBytesBinaryReversible() { + byte[] randomBytes = new byte[1000]; + for (int i = 0; i < 1000; i++) { + Bytes.random(randomBytes); + verifyReversibleForBytes(randomBytes); + } + // some specific cases + verifyReversibleForBytes(new byte[] {}); + verifyReversibleForBytes(new byte[] { '\\', 'x', 'A', 'D' }); + verifyReversibleForBytes(new byte[] { '\\', 'x', 'A', 'D', '\\' }); + } + + private void verifyReversibleForBytes(byte[] originalBytes) { + String convertedString = Bytes.toStringBinary(originalBytes); + byte[] convertedBytes = Bytes.toBytesBinary(convertedString); + if (Bytes.compareTo(originalBytes, convertedBytes) != 0) { + fail("Not reversible for\nbyte[]: " + Arrays.toString(originalBytes) + ",\nStringBinary: " + + convertedString); + } + } + + @Test + public void testStartsWith() { + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("h"))); + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes(""))); + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("hello"))); + assertFalse(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("helloworld"))); + assertFalse(Bytes.startsWith(Bytes.toBytes(""), Bytes.toBytes("hello"))); + } + + @Test + public void testIncrementBytes() { + assertTrue(checkTestIncrementBytes(10, 1)); + assertTrue(checkTestIncrementBytes(12, 123435445)); + assertTrue(checkTestIncrementBytes(124634654, 1)); + assertTrue(checkTestIncrementBytes(10005460, 5005645)); + assertTrue(checkTestIncrementBytes(1, -1)); + assertTrue(checkTestIncrementBytes(10, -1)); + assertTrue(checkTestIncrementBytes(10, -5)); + assertTrue(checkTestIncrementBytes(1005435000, -5)); + assertTrue(checkTestIncrementBytes(10, -43657655)); + assertTrue(checkTestIncrementBytes(-1, 1)); + assertTrue(checkTestIncrementBytes(-26, 5034520)); + assertTrue(checkTestIncrementBytes(-10657200, 5)); + assertTrue(checkTestIncrementBytes(-12343250, 45376475)); + assertTrue(checkTestIncrementBytes(-10, -5)); + assertTrue(checkTestIncrementBytes(-12343250, -5)); + assertTrue(checkTestIncrementBytes(-12, -34565445)); + assertTrue(checkTestIncrementBytes(-1546543452, -34565445)); + } + + private static boolean checkTestIncrementBytes(long val, long amount) { + byte[] value = Bytes.toBytes(val); + byte[] testValue = { -1, -1, -1, -1, -1, -1, -1, -1 }; + if (value[0] > 0) { + testValue = new byte[Bytes.SIZEOF_LONG]; + } + System.arraycopy(value, 0, testValue, testValue.length - value.length, value.length); + + long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount)); + + return (Bytes.toLong(testValue) + amount) == incrementResult; + } + + @Test + public void testFixedSizeString() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + Bytes.writeStringFixedSize(dos, "Hello", 5); + Bytes.writeStringFixedSize(dos, "World", 18); + Bytes.writeStringFixedSize(dos, "", 9); + + try { + // Use a long dash which is three bytes in UTF-8. If encoding happens + // using ISO-8859-1, this will fail. + Bytes.writeStringFixedSize(dos, "Too\u2013Long", 9); + fail("Exception expected"); + } catch (IOException ex) { + assertEquals( + "Trying to write 10 bytes (Too\\xE2\\x80\\x93Long) into a field of " + "length 9", + ex.getMessage()); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + assertEquals("Hello", Bytes.readStringFixedSize(dis, 5)); + assertEquals("World", Bytes.readStringFixedSize(dis, 18)); + assertEquals("", Bytes.readStringFixedSize(dis, 9)); + } + + @Test + public void testCopy() { + byte[] bytes = Bytes.toBytes("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + byte[] copy = Bytes.copy(bytes); + assertNotSame(bytes, copy); + assertTrue(Bytes.equals(bytes, copy)); + } + + @Test + public void testToBytesBinaryTrailingBackslashes() { + try { + Bytes.toBytesBinary("abc\\x00\\x01\\"); + } catch (StringIndexOutOfBoundsException ex) { + fail("Illegal string access: " + ex.getMessage()); + } + } + + @Test + public void testToStringBinary_toBytesBinary_Reversable() { + String bytes = Bytes.toStringBinary(Bytes.toBytes(2.17)); + assertEquals(2.17, Bytes.toDouble(Bytes.toBytesBinary(bytes)), 0); + } + + @Test + public void testUnsignedBinarySearch() { + byte[] bytes = new byte[] { 0, 5, 123, 127, -128, -100, -1 }; + assertEquals(1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 5)); + assertEquals(3, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 127)); + assertEquals(4, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -128)); + assertEquals(5, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -100)); + assertEquals(6, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -1)); + assertEquals(-1 - 1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 2)); + assertEquals(-6 - 1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -5)); + } + + @Test + public void testUnsignedIncrement() { + byte[] a = Bytes.toBytes(0); + int a2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(a), 0); + assertEquals(1, a2); + + byte[] b = Bytes.toBytes(-1); + byte[] actual = Bytes.unsignedCopyAndIncrement(b); + assertNotSame(b, actual); + byte[] expected = new byte[] { 1, 0, 0, 0, 0 }; + assertArrayEquals(expected, actual); + + byte[] c = Bytes.toBytes(255);// should wrap to the next significant byte + int c2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(c), 0); + assertEquals(256, c2); + } + + @Test + public void testIndexOf() { + byte[] array = Bytes.toBytes("hello"); + assertEquals(1, Bytes.indexOf(array, (byte) 'e')); + assertEquals(4, Bytes.indexOf(array, (byte) 'o')); + assertEquals(-1, Bytes.indexOf(array, (byte) 'a')); + assertEquals(0, Bytes.indexOf(array, Bytes.toBytes("hel"))); + assertEquals(2, Bytes.indexOf(array, Bytes.toBytes("ll"))); + assertEquals(-1, Bytes.indexOf(array, Bytes.toBytes("hll"))); + } + + @Test + public void testContains() { + byte[] array = Bytes.toBytes("hello world"); + assertTrue(Bytes.contains(array, (byte) 'e')); + assertTrue(Bytes.contains(array, (byte) 'd')); + assertFalse(Bytes.contains(array, (byte) 'a')); + assertTrue(Bytes.contains(array, Bytes.toBytes("world"))); + assertTrue(Bytes.contains(array, Bytes.toBytes("ello"))); + assertFalse(Bytes.contains(array, Bytes.toBytes("owo"))); + } + + @Test + public void testZero() { + byte[] array = Bytes.toBytes("hello"); + Bytes.zero(array); + for (byte b : array) { + assertEquals(0, b); + } + array = Bytes.toBytes("hello world"); + Bytes.zero(array, 2, 7); + assertFalse(array[0] == 0); + assertFalse(array[1] == 0); + for (int i = 2; i < 9; i++) { + assertEquals(0, array[i]); + } + for (int i = 9; i < array.length; i++) { + assertFalse(array[i] == 0); + } + } + + @Test + public void testPutBuffer() { + byte[] b = new byte[100]; + for (byte i = 0; i < 100; i++) { + Bytes.putByteBuffer(b, i, ByteBuffer.wrap(new byte[] { i })); + } + for (byte i = 0; i < 100; i++) { + assertEquals(i, b[i]); + } + } + + @Test + public void testToFromHex() { + List testStrings = new ArrayList<>(8); + testStrings.addAll(Arrays.asList("", "00", "A0", "ff", "FFffFFFFFFFFFF", "12", + "0123456789abcdef", "283462839463924623984692834692346ABCDFEDDCA0")); + for (String testString : testStrings) { + byte[] byteData = Bytes.fromHex(testString); + assertEquals(testString.length() / 2, byteData.length); + String result = Bytes.toHex(byteData); + assertTrue(testString.equalsIgnoreCase(result)); + } + + List testByteData = new ArrayList<>(5); + testByteData.addAll(Arrays.asList(new byte[0], new byte[1], new byte[10], + new byte[] { 1, 2, 3, 4, 5 }, new byte[] { (byte) 0xFF })); + Random rand = ThreadLocalRandom.current(); + for (int i = 0; i < 20; i++) { + byte[] bytes = new byte[rand.nextInt(100)]; + Bytes.random(bytes); + testByteData.add(bytes); + } + + for (byte[] testData : testByteData) { + String hexString = Bytes.toHex(testData); + assertEquals(testData.length * 2, hexString.length()); + byte[] result = Bytes.fromHex(hexString); + assertArrayEquals(testData, result); + } + } + + @Test + public void testFindCommonPrefix() throws Exception { + // tests for common prefixes less than 8 bytes in length (i.e. using non-vectorized path) + byte[] hello = Bytes.toBytes("hello"); + byte[] helloWorld = Bytes.toBytes("helloworld"); + + assertEquals(5, + Bytes.findCommonPrefix(hello, helloWorld, hello.length, helloWorld.length, 0, 0)); + assertEquals(5, Bytes.findCommonPrefix(hello, hello, hello.length, hello.length, 0, 0)); + assertEquals(3, Bytes.findCommonPrefix(hello, hello, hello.length - 2, hello.length - 2, 2, 2)); + assertEquals(0, Bytes.findCommonPrefix(hello, hello, 0, 0, 0, 0)); + + // tests for common prefixes greater than 8 bytes in length which may use the vectorized path + byte[] hellohello = Bytes.toBytes("hellohello"); + byte[] hellohellohi = Bytes.toBytes("hellohellohi"); + + assertEquals(10, Bytes.findCommonPrefix(hellohello, hellohellohi, hellohello.length, + hellohellohi.length, 0, 0)); + assertEquals(10, Bytes.findCommonPrefix(hellohellohi, hellohello, hellohellohi.length, + hellohello.length, 0, 0)); + assertEquals(10, + Bytes.findCommonPrefix(hellohello, hellohello, hellohello.length, hellohello.length, 0, 0)); + + hellohello[2] = 0; + assertEquals(2, Bytes.findCommonPrefix(hellohello, hellohellohi, hellohello.length, + hellohellohi.length, 0, 0)); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java index 8f180d266c2c..b451d71ed879 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java @@ -17,654 +17,12 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.nio.ByteBuff; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; -import org.apache.hadoop.io.WritableUtils; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@Category({ MiscTests.class, MediumTests.class }) -@RunWith(Parameterized.class) -public class TestByteBufferUtils { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestByteBufferUtils.class); - - private static final String UNSAFE_AVAIL_NAME = "UNSAFE_AVAIL"; - private static final String UNSAFE_UNALIGNED_NAME = "UNSAFE_UNALIGNED"; - private byte[] array; - - @AfterClass - public static void afterClass() throws Exception { - detectAvailabilityOfUnsafe(); - } - - @Parameterized.Parameters - public static Collection parameters() { - return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; - } - - private static void setUnsafe(String fieldName, boolean value) throws Exception { - Field field = ByteBufferUtils.class.getDeclaredField(fieldName); - field.setAccessible(true); - Field modifiersField = ReflectionUtils.getModifiersField(); - modifiersField.setAccessible(true); - int oldModifiers = field.getModifiers(); - modifiersField.setInt(field, oldModifiers & ~Modifier.FINAL); - try { - field.set(null, value); - } finally { - modifiersField.setInt(field, oldModifiers); - } - } - - static void disableUnsafe() throws Exception { - if (ByteBufferUtils.UNSAFE_AVAIL) { - setUnsafe(UNSAFE_AVAIL_NAME, false); - } - if (ByteBufferUtils.UNSAFE_UNALIGNED) { - setUnsafe(UNSAFE_UNALIGNED_NAME, false); - } - assertFalse(ByteBufferUtils.UNSAFE_AVAIL); - assertFalse(ByteBufferUtils.UNSAFE_UNALIGNED); - } - - static void detectAvailabilityOfUnsafe() throws Exception { - if (ByteBufferUtils.UNSAFE_AVAIL != HBasePlatformDependent.isUnsafeAvailable()) { - setUnsafe(UNSAFE_AVAIL_NAME, HBasePlatformDependent.isUnsafeAvailable()); - } - if (ByteBufferUtils.UNSAFE_UNALIGNED != HBasePlatformDependent.unaligned()) { - setUnsafe(UNSAFE_UNALIGNED_NAME, HBasePlatformDependent.unaligned()); - } - assertEquals(ByteBufferUtils.UNSAFE_AVAIL, HBasePlatformDependent.isUnsafeAvailable()); - assertEquals(ByteBufferUtils.UNSAFE_UNALIGNED, HBasePlatformDependent.unaligned()); - } - - public TestByteBufferUtils(boolean useUnsafeIfPossible) throws Exception { - if (useUnsafeIfPossible) { - detectAvailabilityOfUnsafe(); - } else { - disableUnsafe(); - } - } - - /** - * Create an array with sample data. - */ - @Before - public void setUp() { - array = new byte[8]; - for (int i = 0; i < array.length; ++i) { - array[i] = (byte) ('a' + i); - } - } - - private static final int MAX_VLONG_LENGTH = 9; - private static final Collection testNumbers; - - private static void addNumber(Set a, long l) { - if (l != Long.MIN_VALUE) { - a.add(l - 1); - } - a.add(l); - if (l != Long.MAX_VALUE) { - a.add(l + 1); - } - for (long divisor = 3; divisor <= 10; ++divisor) { - for (long delta = -1; delta <= 1; ++delta) { - a.add(l / divisor + delta); - } - } - } - - static { - SortedSet a = new TreeSet<>(); - for (int i = 0; i <= 63; ++i) { - long v = -1L << i; - assertTrue(v < 0); - addNumber(a, v); - v = (1L << i) - 1; - assertTrue(v >= 0); - addNumber(a, v); - } - - testNumbers = Collections.unmodifiableSet(a); - System.err.println("Testing variable-length long serialization using: " + testNumbers - + " (count: " + testNumbers.size() + ")"); - assertEquals(1753, testNumbers.size()); - assertEquals(Long.MIN_VALUE, a.first().longValue()); - assertEquals(Long.MAX_VALUE, a.last().longValue()); - } - - @Test - public void testReadWriteVLong() { - for (long l : testNumbers) { - ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); - ByteBufferUtils.writeVLong(b, l); - b.flip(); - assertEquals(l, ByteBufferUtils.readVLong(b)); - b.flip(); - assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); - } - } - - @Test - public void testReadWriteConsecutiveVLong() { - for (long l : testNumbers) { - ByteBuffer b = ByteBuffer.allocate(2 * MAX_VLONG_LENGTH); - ByteBufferUtils.writeVLong(b, l); - ByteBufferUtils.writeVLong(b, l - 4); - b.flip(); - assertEquals(l, ByteBufferUtils.readVLong(b)); - assertEquals(l - 4, ByteBufferUtils.readVLong(b)); - b.flip(); - assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); - assertEquals(l - 4, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); - } - } - - @Test - public void testConsistencyWithHadoopVLong() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - for (long l : testNumbers) { - baos.reset(); - ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); - ByteBufferUtils.writeVLong(b, l); - String bufStr = Bytes.toStringBinary(b.array(), b.arrayOffset(), b.position()); - WritableUtils.writeVLong(dos, l); - String baosStr = Bytes.toStringBinary(baos.toByteArray()); - assertEquals(baosStr, bufStr); - } - } - - /** - * Test copying to stream from buffer. - */ - @Test - public void testMoveBufferToStream() throws IOException { - final int arrayOffset = 7; - final int initialPosition = 10; - final int endPadding = 5; - byte[] arrayWrapper = new byte[arrayOffset + initialPosition + array.length + endPadding]; - System.arraycopy(array, 0, arrayWrapper, arrayOffset + initialPosition, array.length); - ByteBuffer buffer = - ByteBuffer.wrap(arrayWrapper, arrayOffset, initialPosition + array.length).slice(); - assertEquals(initialPosition + array.length, buffer.limit()); - assertEquals(0, buffer.position()); - buffer.position(initialPosition); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ByteBufferUtils.moveBufferToStream(bos, buffer, array.length); - assertArrayEquals(array, bos.toByteArray()); - assertEquals(initialPosition + array.length, buffer.position()); - } - - /** - * Test copying to stream from buffer with offset. - * @throws IOException On test failure. - */ - @Test - public void testCopyToStreamWithOffset() throws IOException { - ByteBuffer buffer = ByteBuffer.wrap(array); - - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - - ByteBufferUtils.copyBufferToStream(bos, buffer, array.length / 2, array.length / 2); - - byte[] returnedArray = bos.toByteArray(); - for (int i = 0; i < array.length / 2; ++i) { - int pos = array.length / 2 + i; - assertEquals(returnedArray[i], array[pos]); - } - } - - /** - * Test copying data from stream. - * @throws IOException On test failure. - */ - @Test - public void testCopyFromStream() throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(array.length); - ByteArrayInputStream bis = new ByteArrayInputStream(array); - DataInputStream dis = new DataInputStream(bis); - - ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length / 2); - ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length - array.length / 2); - for (int i = 0; i < array.length; ++i) { - assertEquals(array[i], buffer.get(i)); - } - } - - /** - * Test copying from buffer. - */ - @Test - public void testCopyFromBuffer() { - ByteBuffer srcBuffer = ByteBuffer.allocate(array.length); - ByteBuffer dstBuffer = ByteBuffer.allocate(array.length); - srcBuffer.put(array); - - ByteBufferUtils.copyFromBufferToBuffer(srcBuffer, dstBuffer, array.length / 2, - array.length / 4); - for (int i = 0; i < array.length / 4; ++i) { - assertEquals(srcBuffer.get(i + array.length / 2), dstBuffer.get(i)); - } - } - - /** - * Test 7-bit encoding of integers. - * @throws IOException On test failure. - */ - @Test - public void testCompressedInt() throws IOException { - testCompressedInt(0); - testCompressedInt(Integer.MAX_VALUE); - testCompressedInt(Integer.MIN_VALUE); - - for (int i = 0; i < 3; i++) { - testCompressedInt((128 << i) - 1); - } - - for (int i = 0; i < 3; i++) { - testCompressedInt((128 << i)); - } - } - - /** - * Test how much bytes we need to store integer. - */ - @Test - public void testIntFitsIn() { - assertEquals(1, ByteBufferUtils.intFitsIn(0)); - assertEquals(1, ByteBufferUtils.intFitsIn(1)); - assertEquals(2, ByteBufferUtils.intFitsIn(1 << 8)); - assertEquals(3, ByteBufferUtils.intFitsIn(1 << 16)); - assertEquals(4, ByteBufferUtils.intFitsIn(-1)); - assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MAX_VALUE)); - assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MIN_VALUE)); - } - - /** - * Test how much bytes we need to store long. - */ - @Test - public void testLongFitsIn() { - assertEquals(1, ByteBufferUtils.longFitsIn(0)); - assertEquals(1, ByteBufferUtils.longFitsIn(1)); - assertEquals(3, ByteBufferUtils.longFitsIn(1L << 16)); - assertEquals(5, ByteBufferUtils.longFitsIn(1L << 32)); - assertEquals(8, ByteBufferUtils.longFitsIn(-1)); - assertEquals(8, ByteBufferUtils.longFitsIn(Long.MIN_VALUE)); - assertEquals(8, ByteBufferUtils.longFitsIn(Long.MAX_VALUE)); - } - - /** - * Test if we are comparing equal bytes. - */ - @Test - public void testArePartEqual() { - byte[] array = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4 }; - ByteBuffer buffer = ByteBuffer.wrap(array); - assertTrue(ByteBufferUtils.arePartsEqual(buffer, 0, 4, 5, 4)); - assertTrue(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 2)); - assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 3)); - assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 3, 6, 2)); - assertFalse(ByteBufferUtils.arePartsEqual(buffer, 0, 3, 6, 3)); - } - - /** - * Test serializing int to bytes - */ - @Test - public void testPutInt() { - testPutInt(0); - testPutInt(Integer.MAX_VALUE); - - for (int i = 0; i < 3; i++) { - testPutInt((128 << i) - 1); - } - - for (int i = 0; i < 3; i++) { - testPutInt((128 << i)); - } - } - - // Utility methods invoked from test methods - - private void testCompressedInt(int value) throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ByteBufferUtils.putCompressedInt(bos, value); - ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); - int parsedValue = ByteBufferUtils.readCompressedInt(bis); - assertEquals(value, parsedValue); - } - - private void testPutInt(int value) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try { - ByteBufferUtils.putInt(baos, value); - } catch (IOException e) { - throw new RuntimeException("Bug in putIn()", e); - } - - ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); - DataInputStream dis = new DataInputStream(bais); - try { - assertEquals(dis.readInt(), value); - } catch (IOException e) { - throw new RuntimeException("Bug in test!", e); - } - } - - @Test - public void testToBytes() { - ByteBuffer buffer = ByteBuffer.allocate(5); - buffer.put(new byte[] { 0, 1, 2, 3, 4 }); - assertEquals(5, buffer.position()); - assertEquals(5, buffer.limit()); - byte[] copy = ByteBufferUtils.toBytes(buffer, 2); - assertArrayEquals(new byte[] { 2, 3, 4 }, copy); - assertEquals(5, buffer.position()); - assertEquals(5, buffer.limit()); - } - - @Test - public void testToPrimitiveTypes() { - ByteBuffer buffer = ByteBuffer.allocate(15); - long l = 988L; - int i = 135; - short s = 7; - buffer.putLong(l); - buffer.putShort(s); - buffer.putInt(i); - assertEquals(l, ByteBufferUtils.toLong(buffer, 0)); - assertEquals(s, ByteBufferUtils.toShort(buffer, 8)); - assertEquals(i, ByteBufferUtils.toInt(buffer, 10)); - } - - @Test - public void testCopyFromArrayToBuffer() { - byte[] b = new byte[15]; - b[0] = -1; - long l = 988L; - int i = 135; - short s = 7; - Bytes.putLong(b, 1, l); - Bytes.putShort(b, 9, s); - Bytes.putInt(b, 11, i); - ByteBuffer buffer = ByteBuffer.allocate(14); - ByteBufferUtils.copyFromArrayToBuffer(buffer, b, 1, 14); - buffer.rewind(); - assertEquals(l, buffer.getLong()); - assertEquals(s, buffer.getShort()); - assertEquals(i, buffer.getInt()); - } - - private void testCopyFromSrcToDestWithThreads(Object input, Object output, List lengthes, - List offsets) throws InterruptedException { - assertTrue((input instanceof ByteBuffer) || (input instanceof byte[])); - assertTrue((output instanceof ByteBuffer) || (output instanceof byte[])); - assertEquals(lengthes.size(), offsets.size()); - - final int threads = lengthes.size(); - CountDownLatch latch = new CountDownLatch(1); - List exes = new ArrayList<>(threads); - int oldInputPos = (input instanceof ByteBuffer) ? ((ByteBuffer) input).position() : 0; - int oldOutputPos = (output instanceof ByteBuffer) ? ((ByteBuffer) output).position() : 0; - for (int i = 0; i != threads; ++i) { - int offset = offsets.get(i); - int length = lengthes.get(i); - exes.add(() -> { - try { - latch.await(); - if (input instanceof ByteBuffer && output instanceof byte[]) { - ByteBufferUtils.copyFromBufferToArray((byte[]) output, (ByteBuffer) input, offset, - offset, length); - } - if (input instanceof byte[] && output instanceof ByteBuffer) { - ByteBufferUtils.copyFromArrayToBuffer((ByteBuffer) output, offset, (byte[]) input, - offset, length); - } - if (input instanceof ByteBuffer && output instanceof ByteBuffer) { - ByteBufferUtils.copyFromBufferToBuffer((ByteBuffer) input, (ByteBuffer) output, offset, - offset, length); - } - } catch (InterruptedException ex) { - throw new RuntimeException(ex); - } - }); - } - ExecutorService service = Executors.newFixedThreadPool(threads); - exes.forEach(service::execute); - latch.countDown(); - service.shutdown(); - assertTrue(service.awaitTermination(5, TimeUnit.SECONDS)); - if (input instanceof ByteBuffer) { - assertEquals(oldInputPos, ((ByteBuffer) input).position()); - } - if (output instanceof ByteBuffer) { - assertEquals(oldOutputPos, ((ByteBuffer) output).position()); - } - String inputString = (input instanceof ByteBuffer) - ? Bytes.toString(Bytes.toBytes((ByteBuffer) input)) - : Bytes.toString((byte[]) input); - String outputString = (output instanceof ByteBuffer) - ? Bytes.toString(Bytes.toBytes((ByteBuffer) output)) - : Bytes.toString((byte[]) output); - assertEquals(inputString, outputString); - } - - @Test - public void testCopyFromSrcToDestWithThreads() throws InterruptedException { - List words = - Arrays.asList(Bytes.toBytes("with"), Bytes.toBytes("great"), Bytes.toBytes("power"), - Bytes.toBytes("comes"), Bytes.toBytes("great"), Bytes.toBytes("responsibility")); - List lengthes = words.stream().map(v -> v.length).collect(Collectors.toList()); - List offsets = new ArrayList<>(words.size()); - for (int i = 0; i != words.size(); ++i) { - offsets.add(words.subList(0, i).stream().mapToInt(v -> v.length).sum()); - } - - int totalSize = words.stream().mapToInt(v -> v.length).sum(); - byte[] fullContent = new byte[totalSize]; - int offset = 0; - for (byte[] w : words) { - offset = Bytes.putBytes(fullContent, offset, w, 0, w.length); - } - - // test copyFromBufferToArray - for (ByteBuffer input : Arrays.asList(ByteBuffer.allocateDirect(totalSize), - ByteBuffer.allocate(totalSize))) { - words.forEach(input::put); - byte[] output = new byte[totalSize]; - testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); - } - - // test copyFromArrayToBuffer - for (ByteBuffer output : Arrays.asList(ByteBuffer.allocateDirect(totalSize), - ByteBuffer.allocate(totalSize))) { - byte[] input = fullContent; - testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); - } - - // test copyFromBufferToBuffer - for (ByteBuffer input : Arrays.asList(ByteBuffer.allocateDirect(totalSize), - ByteBuffer.allocate(totalSize))) { - words.forEach(input::put); - for (ByteBuffer output : Arrays.asList(ByteBuffer.allocateDirect(totalSize), - ByteBuffer.allocate(totalSize))) { - testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); - } - } - } - - @Test - public void testCopyFromBufferToArray() { - ByteBuffer buffer = ByteBuffer.allocate(15); - buffer.put((byte) -1); - long l = 988L; - int i = 135; - short s = 7; - buffer.putShort(s); - buffer.putInt(i); - buffer.putLong(l); - byte[] b = new byte[15]; - ByteBufferUtils.copyFromBufferToArray(b, buffer, 1, 1, 14); - assertEquals(s, Bytes.toShort(b, 1)); - assertEquals(i, Bytes.toInt(b, 3)); - assertEquals(l, Bytes.toLong(b, 7)); - } - - @Test - public void testRelativeCopyFromBuffertoBuffer() { - ByteBuffer bb1 = ByteBuffer.allocate(135); - ByteBuffer bb2 = ByteBuffer.allocate(135); - fillBB(bb1, (byte) 5); - ByteBufferUtils.copyFromBufferToBuffer(bb1, bb2); - assertTrue(bb1.position() == bb2.position()); - assertTrue(bb1.limit() == bb2.limit()); - bb1 = ByteBuffer.allocateDirect(135); - bb2 = ByteBuffer.allocateDirect(135); - fillBB(bb1, (byte) 5); - ByteBufferUtils.copyFromBufferToBuffer(bb1, bb2); - assertTrue(bb1.position() == bb2.position()); - assertTrue(bb1.limit() == bb2.limit()); - } - - @Test - public void testCompareTo() { - ByteBuffer bb1 = ByteBuffer.allocate(135); - ByteBuffer bb2 = ByteBuffer.allocate(135); - byte[] b = new byte[71]; - fillBB(bb1, (byte) 5); - fillBB(bb2, (byte) 5); - fillArray(b, (byte) 5); - assertEquals(0, ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); - assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), b, 0, b.length) > 0); - bb2.put(134, (byte) 6); - assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()) < 0); - bb2.put(6, (byte) 4); - assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()) > 0); - // Assert reverse comparing BB and bytearray works. - ByteBuffer bb3 = ByteBuffer.allocate(135); - fillBB(bb3, (byte) 0); - byte[] b3 = new byte[135]; - fillArray(b3, (byte) 1); - int result = ByteBufferUtils.compareTo(b3, 0, b3.length, bb3, 0, bb3.remaining()); - assertTrue(result > 0); - result = ByteBufferUtils.compareTo(bb3, 0, bb3.remaining(), b3, 0, b3.length); - assertTrue(result < 0); - byte[] b4 = Bytes.toBytes("123"); - ByteBuffer bb4 = ByteBuffer.allocate(10 + b4.length); - for (int i = 10; i < bb4.capacity(); ++i) { - bb4.put(i, b4[i - 10]); - } - result = ByteBufferUtils.compareTo(b4, 0, b4.length, bb4, 10, b4.length); - assertEquals(0, result); - } - - @Test - public void testEquals() { - byte[] a = Bytes.toBytes("http://A"); - ByteBuffer bb = ByteBuffer.wrap(a); - - assertTrue(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, - HConstants.EMPTY_BYTE_BUFFER, 0, 0)); - - assertFalse(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, bb, 0, a.length)); - - assertFalse(ByteBufferUtils.equals(bb, 0, 0, HConstants.EMPTY_BYTE_BUFFER, 0, a.length)); - - assertTrue(ByteBufferUtils.equals(bb, 0, a.length, bb, 0, a.length)); - - assertTrue(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, - HConstants.EMPTY_BYTE_ARRAY, 0, 0)); - - assertFalse(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, a, 0, a.length)); - - assertFalse(ByteBufferUtils.equals(bb, 0, a.length, HConstants.EMPTY_BYTE_ARRAY, 0, 0)); - - assertTrue(ByteBufferUtils.equals(bb, 0, a.length, a, 0, a.length)); - } - - @Test - public void testFindCommonPrefix() { - ByteBuffer bb1 = ByteBuffer.allocate(135); - ByteBuffer bb2 = ByteBuffer.allocate(135); - ByteBuffer bb3 = ByteBuffer.allocateDirect(135); - byte[] b = new byte[71]; - - fillBB(bb1, (byte) 5); - fillBB(bb2, (byte) 5); - fillBB(bb3, (byte) 5); - fillArray(b, (byte) 5); - - assertEquals(135, - ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); - assertEquals(71, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length)); - assertEquals(135, - ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb3, 0, bb3.remaining())); - assertEquals(71, ByteBufferUtils.findCommonPrefix(bb3, 0, bb3.remaining(), b, 0, b.length)); - - b[13] = 9; - assertEquals(13, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length)); - - bb2.put(134, (byte) 6); - assertEquals(134, - ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); - - bb2.put(6, (byte) 4); - assertEquals(6, - ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); - } - - private static void fillBB(ByteBuffer bb, byte b) { - for (int i = bb.position(); i < bb.limit(); i++) { - bb.put(i, b); - } - } +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; - private static void fillArray(byte[] bb, byte b) { - for (int i = 0; i < bb.length; i++) { - bb[i] = b; - } - } +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestByteBufferUtils extends ByteBufferUtilsTestBase { } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtilsWoUnsafe.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtilsWoUnsafe.java new file mode 100644 index 000000000000..c02db2142c1c --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtilsWoUnsafe.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mockStatic; + +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.mockito.MockedStatic; + +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestByteBufferUtilsWoUnsafe extends ByteBufferUtilsTestBase { + + @BeforeAll + public static void disableUnsafe() { + try (MockedStatic mocked = mockStatic(HBasePlatformDependent.class)) { + mocked.when(HBasePlatformDependent::isUnsafeAvailable).thenReturn(false); + mocked.when(HBasePlatformDependent::unaligned).thenReturn(false); + assertFalse(ByteBufferUtils.UNSAFE_AVAIL); + assertFalse(ByteBufferUtils.UNSAFE_UNALIGNED); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java index b74348959982..0122e91d7ea9 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java @@ -17,615 +17,12 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; -import org.apache.hadoop.io.WritableUtils; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({ MiscTests.class, MediumTests.class }) -public class TestBytes { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestBytes.class); - - private static void setUnsafe(boolean value) throws Exception { - Field field = Bytes.class.getDeclaredField("UNSAFE_UNALIGNED"); - field.setAccessible(true); - - Field modifiersField = ReflectionUtils.getModifiersField(); - modifiersField.setAccessible(true); - int oldModifiers = field.getModifiers(); - modifiersField.setInt(field, oldModifiers & ~Modifier.FINAL); - try { - field.set(null, value); - } finally { - modifiersField.setInt(field, oldModifiers); - } - assertEquals(Bytes.UNSAFE_UNALIGNED, value); - } - - @Test - public void testShort() throws Exception { - testShort(false); - } - - @Test - public void testShortUnsafe() throws Exception { - testShort(true); - } - - private static void testShort(boolean unsafe) throws Exception { - setUnsafe(unsafe); - try { - for (short n : Arrays.asList(Short.MIN_VALUE, (short) -100, (short) -1, (short) 0, (short) 1, - (short) 300, Short.MAX_VALUE)) { - byte[] bytes = Bytes.toBytes(n); - assertEquals(Bytes.toShort(bytes, 0, bytes.length), n); - } - } finally { - setUnsafe(HBasePlatformDependent.unaligned()); - } - } - - @Test - public void testNullHashCode() { - byte[] b = null; - Exception ee = null; - try { - Bytes.hashCode(b); - } catch (Exception e) { - ee = e; - } - assertNotNull(ee); - } - - @Test - public void testAdd() { - byte[] a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - byte[] b = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - byte[] c = { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }; - byte[] result1 = Bytes.add(a, b, c); - byte[] result2 = Bytes.add(new byte[][] { a, b, c }); - assertEquals(0, Bytes.compareTo(result1, result2)); - } - - @Test - public void testSplit() { - byte[] lowest = Bytes.toBytes("AAA"); - byte[] middle = Bytes.toBytes("CCC"); - byte[] highest = Bytes.toBytes("EEE"); - byte[][] parts = Bytes.split(lowest, highest, 1); - for (byte[] bytes : parts) { - System.out.println(Bytes.toString(bytes)); - } - assertEquals(3, parts.length); - assertTrue(Bytes.equals(parts[1], middle)); - // Now divide into three parts. Change highest so split is even. - highest = Bytes.toBytes("DDD"); - parts = Bytes.split(lowest, highest, 2); - for (byte[] part : parts) { - System.out.println(Bytes.toString(part)); - } - assertEquals(4, parts.length); - // Assert that 3rd part is 'CCC'. - assertTrue(Bytes.equals(parts[2], middle)); - } - - @Test - public void testSplit2() { - // More split tests. - byte[] lowest = Bytes.toBytes("http://A"); - byte[] highest = Bytes.toBytes("http://z"); - byte[] middle = Bytes.toBytes("http://]"); - byte[][] parts = Bytes.split(lowest, highest, 1); - for (byte[] part : parts) { - System.out.println(Bytes.toString(part)); - } - assertEquals(3, parts.length); - assertTrue(Bytes.equals(parts[1], middle)); - } - - @Test - public void testSplit3() { - // Test invalid split cases - byte[] low = { 1, 1, 1 }; - byte[] high = { 1, 1, 3 }; - - // If swapped, should throw IAE - try { - Bytes.split(high, low, 1); - fail("Should not be able to split if low > high"); - } catch (IllegalArgumentException iae) { - // Correct - } - - // Single split should work - byte[][] parts = Bytes.split(low, high, 1); - for (int i = 0; i < parts.length; i++) { - System.out.println("" + i + " -> " + Bytes.toStringBinary(parts[i])); - } - assertEquals("Returned split should have 3 parts but has " + parts.length, 3, parts.length); - - // If split more than once, use additional byte to split - parts = Bytes.split(low, high, 2); - assertNotNull("Split with an additional byte", parts); - assertEquals(parts.length, low.length + 1); - - // Split 0 times should throw IAE - try { - Bytes.split(low, high, 0); - fail("Should not be able to split 0 times"); - } catch (IllegalArgumentException iae) { - // Correct - } - } - - @Test - public void testToInt() { - int[] ints = { -1, 123, Integer.MIN_VALUE, Integer.MAX_VALUE }; - for (int anInt : ints) { - byte[] b = Bytes.toBytes(anInt); - assertEquals(anInt, Bytes.toInt(b)); - byte[] b2 = bytesWithOffset(b); - assertEquals(anInt, Bytes.toInt(b2, 1)); - assertEquals(anInt, Bytes.toInt(b2, 1, Bytes.SIZEOF_INT)); - } - } - - @Test - public void testToLong() { - long[] longs = { -1L, 123L, Long.MIN_VALUE, Long.MAX_VALUE }; - for (long aLong : longs) { - byte[] b = Bytes.toBytes(aLong); - assertEquals(aLong, Bytes.toLong(b)); - byte[] b2 = bytesWithOffset(b); - assertEquals(aLong, Bytes.toLong(b2, 1)); - assertEquals(aLong, Bytes.toLong(b2, 1, Bytes.SIZEOF_LONG)); - } - } - - @Test - public void testToFloat() { - float[] floats = { -1f, 123.123f, Float.MAX_VALUE }; - for (float aFloat : floats) { - byte[] b = Bytes.toBytes(aFloat); - assertEquals(aFloat, Bytes.toFloat(b), 0.0f); - byte[] b2 = bytesWithOffset(b); - assertEquals(aFloat, Bytes.toFloat(b2, 1), 0.0f); - } - } - - @Test - public void testToDouble() { - double[] doubles = { Double.MIN_VALUE, Double.MAX_VALUE }; - for (double aDouble : doubles) { - byte[] b = Bytes.toBytes(aDouble); - assertEquals(aDouble, Bytes.toDouble(b), 0.0); - byte[] b2 = bytesWithOffset(b); - assertEquals(aDouble, Bytes.toDouble(b2, 1), 0.0); - } - } - - @Test - public void testToBigDecimal() { - BigDecimal[] decimals = - { new BigDecimal("-1"), new BigDecimal("123.123"), new BigDecimal("123123123123") }; - for (BigDecimal decimal : decimals) { - byte[] b = Bytes.toBytes(decimal); - assertEquals(decimal, Bytes.toBigDecimal(b)); - byte[] b2 = bytesWithOffset(b); - assertEquals(decimal, Bytes.toBigDecimal(b2, 1, b.length)); - } - } - - private byte[] bytesWithOffset(byte[] src) { - // add one byte in front to test offset - byte[] result = new byte[src.length + 1]; - result[0] = (byte) 0xAA; - System.arraycopy(src, 0, result, 1, src.length); - return result; - } - - @Test - public void testToBytesForByteBuffer() { - byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - ByteBuffer target = ByteBuffer.wrap(array); - target.position(2); - target.limit(7); - - byte[] actual = Bytes.toBytes(target); - byte[] expected = { 0, 1, 2, 3, 4, 5, 6 }; - assertArrayEquals(expected, actual); - assertEquals(2, target.position()); - assertEquals(7, target.limit()); - - ByteBuffer target2 = target.slice(); - assertEquals(0, target2.position()); - assertEquals(5, target2.limit()); - - byte[] actual2 = Bytes.toBytes(target2); - byte[] expected2 = { 2, 3, 4, 5, 6 }; - assertArrayEquals(expected2, actual2); - assertEquals(0, target2.position()); - assertEquals(5, target2.limit()); - } - - @Test - public void testGetBytesForByteBuffer() { - byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - ByteBuffer target = ByteBuffer.wrap(array); - target.position(2); - target.limit(7); - - byte[] actual = Bytes.getBytes(target); - byte[] expected = { 2, 3, 4, 5, 6 }; - assertArrayEquals(expected, actual); - assertEquals(2, target.position()); - assertEquals(7, target.limit()); - } - - @Test - public void testReadAsVLong() throws Exception { - long[] longs = { -1L, 123L, Long.MIN_VALUE, Long.MAX_VALUE }; - for (long aLong : longs) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream output = new DataOutputStream(baos); - WritableUtils.writeVLong(output, aLong); - byte[] long_bytes_no_offset = baos.toByteArray(); - assertEquals(aLong, Bytes.readAsVLong(long_bytes_no_offset, 0)); - byte[] long_bytes_with_offset = bytesWithOffset(long_bytes_no_offset); - assertEquals(aLong, Bytes.readAsVLong(long_bytes_with_offset, 1)); - } - } - - @Test - public void testToStringBinaryForBytes() { - byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; - String actual = Bytes.toStringBinary(array); - String expected = "09azAZ@\\x01"; - assertEquals(expected, actual); - - String actual2 = Bytes.toStringBinary(array, 2, 3); - String expected2 = "azA"; - assertEquals(expected2, actual2); - } - - @Test - public void testToStringBinaryForArrayBasedByteBuffer() { - byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; - ByteBuffer target = ByteBuffer.wrap(array); - String actual = Bytes.toStringBinary(target); - String expected = "09azAZ@\\x01"; - assertEquals(expected, actual); - } - - @Test - public void testToStringBinaryForReadOnlyByteBuffer() { - byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; - ByteBuffer target = ByteBuffer.wrap(array).asReadOnlyBuffer(); - String actual = Bytes.toStringBinary(target); - String expected = "09azAZ@\\x01"; - assertEquals(expected, actual); - } - - @Test - public void testBinarySearch() { - byte[][] arr = { { 1 }, { 3 }, { 5 }, { 7 }, { 9 }, { 11 }, { 13 }, { 15 }, }; - byte[] key1 = { 3, 1 }; - byte[] key2 = { 4, 9 }; - byte[] key2_2 = { 4 }; - byte[] key3 = { 5, 11 }; - byte[] key4 = { 0 }; - byte[] key5 = { 2 }; - - assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1)); - assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1)); - assertEquals(-(2 + 1), Arrays.binarySearch(arr, key2_2, Bytes.BYTES_COMPARATOR)); - assertEquals(-(2 + 1), Bytes.binarySearch(arr, key2, 0, 1)); - assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1)); - assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1)); - assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1)); - assertEquals(-1, Bytes.binarySearch(arr, key4, 0, 1)); - assertEquals(-2, Bytes.binarySearch(arr, key5, 0, 1)); - - // Search for values to the left and to the right of each item in the array. - for (int i = 0; i < arr.length; ++i) { - assertEquals(-(i + 1), Bytes.binarySearch(arr, new byte[] { (byte) (arr[i][0] - 1) }, 0, 1)); - assertEquals(-(i + 2), Bytes.binarySearch(arr, new byte[] { (byte) (arr[i][0] + 1) }, 0, 1)); - } - } - - @Test - public void testToStringBytesBinaryReversible() { - byte[] randomBytes = new byte[1000]; - for (int i = 0; i < 1000; i++) { - Bytes.random(randomBytes); - verifyReversibleForBytes(randomBytes); - } - // some specific cases - verifyReversibleForBytes(new byte[] {}); - verifyReversibleForBytes(new byte[] { '\\', 'x', 'A', 'D' }); - verifyReversibleForBytes(new byte[] { '\\', 'x', 'A', 'D', '\\' }); - } - - private void verifyReversibleForBytes(byte[] originalBytes) { - String convertedString = Bytes.toStringBinary(originalBytes); - byte[] convertedBytes = Bytes.toBytesBinary(convertedString); - if (Bytes.compareTo(originalBytes, convertedBytes) != 0) { - fail("Not reversible for\nbyte[]: " + Arrays.toString(originalBytes) + ",\nStringBinary: " - + convertedString); - } - } - - @Test - public void testStartsWith() { - assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("h"))); - assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes(""))); - assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("hello"))); - assertFalse(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("helloworld"))); - assertFalse(Bytes.startsWith(Bytes.toBytes(""), Bytes.toBytes("hello"))); - } - - @Test - public void testIncrementBytes() { - assertTrue(checkTestIncrementBytes(10, 1)); - assertTrue(checkTestIncrementBytes(12, 123435445)); - assertTrue(checkTestIncrementBytes(124634654, 1)); - assertTrue(checkTestIncrementBytes(10005460, 5005645)); - assertTrue(checkTestIncrementBytes(1, -1)); - assertTrue(checkTestIncrementBytes(10, -1)); - assertTrue(checkTestIncrementBytes(10, -5)); - assertTrue(checkTestIncrementBytes(1005435000, -5)); - assertTrue(checkTestIncrementBytes(10, -43657655)); - assertTrue(checkTestIncrementBytes(-1, 1)); - assertTrue(checkTestIncrementBytes(-26, 5034520)); - assertTrue(checkTestIncrementBytes(-10657200, 5)); - assertTrue(checkTestIncrementBytes(-12343250, 45376475)); - assertTrue(checkTestIncrementBytes(-10, -5)); - assertTrue(checkTestIncrementBytes(-12343250, -5)); - assertTrue(checkTestIncrementBytes(-12, -34565445)); - assertTrue(checkTestIncrementBytes(-1546543452, -34565445)); - } - - private static boolean checkTestIncrementBytes(long val, long amount) { - byte[] value = Bytes.toBytes(val); - byte[] testValue = { -1, -1, -1, -1, -1, -1, -1, -1 }; - if (value[0] > 0) { - testValue = new byte[Bytes.SIZEOF_LONG]; - } - System.arraycopy(value, 0, testValue, testValue.length - value.length, value.length); - - long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount)); - - return (Bytes.toLong(testValue) + amount) == incrementResult; - } - - @Test - public void testFixedSizeString() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - Bytes.writeStringFixedSize(dos, "Hello", 5); - Bytes.writeStringFixedSize(dos, "World", 18); - Bytes.writeStringFixedSize(dos, "", 9); - - try { - // Use a long dash which is three bytes in UTF-8. If encoding happens - // using ISO-8859-1, this will fail. - Bytes.writeStringFixedSize(dos, "Too\u2013Long", 9); - fail("Exception expected"); - } catch (IOException ex) { - assertEquals( - "Trying to write 10 bytes (Too\\xE2\\x80\\x93Long) into a field of " + "length 9", - ex.getMessage()); - } - - ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); - DataInputStream dis = new DataInputStream(bais); - assertEquals("Hello", Bytes.readStringFixedSize(dis, 5)); - assertEquals("World", Bytes.readStringFixedSize(dis, 18)); - assertEquals("", Bytes.readStringFixedSize(dis, 9)); - } - - @Test - public void testCopy() { - byte[] bytes = Bytes.toBytes("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - byte[] copy = Bytes.copy(bytes); - assertNotSame(bytes, copy); - assertTrue(Bytes.equals(bytes, copy)); - } - - @Test - public void testToBytesBinaryTrailingBackslashes() { - try { - Bytes.toBytesBinary("abc\\x00\\x01\\"); - } catch (StringIndexOutOfBoundsException ex) { - fail("Illegal string access: " + ex.getMessage()); - } - } - - @Test - public void testToStringBinary_toBytesBinary_Reversable() { - String bytes = Bytes.toStringBinary(Bytes.toBytes(2.17)); - assertEquals(2.17, Bytes.toDouble(Bytes.toBytesBinary(bytes)), 0); - } - - @Test - public void testUnsignedBinarySearch() { - byte[] bytes = new byte[] { 0, 5, 123, 127, -128, -100, -1 }; - Assert.assertEquals(1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 5)); - Assert.assertEquals(3, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 127)); - Assert.assertEquals(4, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -128)); - Assert.assertEquals(5, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -100)); - Assert.assertEquals(6, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -1)); - Assert.assertEquals(-1 - 1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 2)); - Assert.assertEquals(-6 - 1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -5)); - } - - @Test - public void testUnsignedIncrement() { - byte[] a = Bytes.toBytes(0); - int a2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(a), 0); - Assert.assertEquals(1, a2); - - byte[] b = Bytes.toBytes(-1); - byte[] actual = Bytes.unsignedCopyAndIncrement(b); - Assert.assertNotSame(b, actual); - byte[] expected = new byte[] { 1, 0, 0, 0, 0 }; - assertArrayEquals(expected, actual); - - byte[] c = Bytes.toBytes(255);// should wrap to the next significant byte - int c2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(c), 0); - Assert.assertEquals(256, c2); - } - - @Test - public void testIndexOf() { - byte[] array = Bytes.toBytes("hello"); - assertEquals(1, Bytes.indexOf(array, (byte) 'e')); - assertEquals(4, Bytes.indexOf(array, (byte) 'o')); - assertEquals(-1, Bytes.indexOf(array, (byte) 'a')); - assertEquals(0, Bytes.indexOf(array, Bytes.toBytes("hel"))); - assertEquals(2, Bytes.indexOf(array, Bytes.toBytes("ll"))); - assertEquals(-1, Bytes.indexOf(array, Bytes.toBytes("hll"))); - } - - @Test - public void testContains() { - byte[] array = Bytes.toBytes("hello world"); - assertTrue(Bytes.contains(array, (byte) 'e')); - assertTrue(Bytes.contains(array, (byte) 'd')); - assertFalse(Bytes.contains(array, (byte) 'a')); - assertTrue(Bytes.contains(array, Bytes.toBytes("world"))); - assertTrue(Bytes.contains(array, Bytes.toBytes("ello"))); - assertFalse(Bytes.contains(array, Bytes.toBytes("owo"))); - } - - @Test - public void testZero() { - byte[] array = Bytes.toBytes("hello"); - Bytes.zero(array); - for (byte b : array) { - assertEquals(0, b); - } - array = Bytes.toBytes("hello world"); - Bytes.zero(array, 2, 7); - assertFalse(array[0] == 0); - assertFalse(array[1] == 0); - for (int i = 2; i < 9; i++) { - assertEquals(0, array[i]); - } - for (int i = 9; i < array.length; i++) { - assertFalse(array[i] == 0); - } - } - - @Test - public void testPutBuffer() { - byte[] b = new byte[100]; - for (byte i = 0; i < 100; i++) { - Bytes.putByteBuffer(b, i, ByteBuffer.wrap(new byte[] { i })); - } - for (byte i = 0; i < 100; i++) { - Assert.assertEquals(i, b[i]); - } - } - - @Test - public void testToFromHex() { - List testStrings = new ArrayList<>(8); - testStrings.addAll(Arrays.asList("", "00", "A0", "ff", "FFffFFFFFFFFFF", "12", - "0123456789abcdef", "283462839463924623984692834692346ABCDFEDDCA0")); - for (String testString : testStrings) { - byte[] byteData = Bytes.fromHex(testString); - Assert.assertEquals(testString.length() / 2, byteData.length); - String result = Bytes.toHex(byteData); - Assert.assertTrue(testString.equalsIgnoreCase(result)); - } - - List testByteData = new ArrayList<>(5); - testByteData.addAll(Arrays.asList(new byte[0], new byte[1], new byte[10], - new byte[] { 1, 2, 3, 4, 5 }, new byte[] { (byte) 0xFF })); - Random rand = ThreadLocalRandom.current(); - for (int i = 0; i < 20; i++) { - byte[] bytes = new byte[rand.nextInt(100)]; - Bytes.random(bytes); - testByteData.add(bytes); - } - - for (byte[] testData : testByteData) { - String hexString = Bytes.toHex(testData); - Assert.assertEquals(testData.length * 2, hexString.length()); - byte[] result = Bytes.fromHex(hexString); - assertArrayEquals(testData, result); - } - } - - @Test - public void testFindCommonPrefix() throws Exception { - testFindCommonPrefix(false); - } - - @Test - public void testFindCommonPrefixUnsafe() throws Exception { - testFindCommonPrefix(true); - } - - private static void testFindCommonPrefix(boolean unsafe) throws Exception { - setUnsafe(unsafe); - try { - // tests for common prefixes less than 8 bytes in length (i.e. using non-vectorized path) - byte[] hello = Bytes.toBytes("hello"); - byte[] helloWorld = Bytes.toBytes("helloworld"); - - assertEquals(5, - Bytes.findCommonPrefix(hello, helloWorld, hello.length, helloWorld.length, 0, 0)); - assertEquals(5, Bytes.findCommonPrefix(hello, hello, hello.length, hello.length, 0, 0)); - assertEquals(3, - Bytes.findCommonPrefix(hello, hello, hello.length - 2, hello.length - 2, 2, 2)); - assertEquals(0, Bytes.findCommonPrefix(hello, hello, 0, 0, 0, 0)); - - // tests for common prefixes greater than 8 bytes in length which may use the vectorized path - byte[] hellohello = Bytes.toBytes("hellohello"); - byte[] hellohellohi = Bytes.toBytes("hellohellohi"); +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; - assertEquals(10, Bytes.findCommonPrefix(hellohello, hellohellohi, hellohello.length, - hellohellohi.length, 0, 0)); - assertEquals(10, Bytes.findCommonPrefix(hellohellohi, hellohello, hellohellohi.length, - hellohello.length, 0, 0)); - assertEquals(10, - Bytes.findCommonPrefix(hellohello, hellohello, hellohello.length, hellohello.length, 0, 0)); +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestBytes extends BytesTestBase { - hellohello[2] = 0; - assertEquals(2, Bytes.findCommonPrefix(hellohello, hellohellohi, hellohello.length, - hellohellohi.length, 0, 0)); - } finally { - setUnsafe(HBasePlatformDependent.unaligned()); - } - } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytesWoUnsafe.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytesWoUnsafe.java new file mode 100644 index 000000000000..8aacab4b8514 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytesWoUnsafe.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mockStatic; + +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.mockito.MockedStatic; + +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestBytesWoUnsafe extends BytesTestBase { + + @BeforeAll + public static void disableUnsafe() { + try (MockedStatic mocked = mockStatic(HBasePlatformDependent.class)) { + mocked.when(HBasePlatformDependent::unaligned).thenReturn(false); + assertFalse(Bytes.UNSAFE_UNALIGNED); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastStringPool.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastStringPool.java new file mode 100644 index 000000000000..dd0a83393342 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastStringPool.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag(SmallTests.TAG) +public class TestFastStringPool { + + private static final Logger LOG = LoggerFactory.getLogger(TestFastStringPool.class); + + @Test + public void testMultiThread() throws InterruptedException { + FastStringPool pool = new FastStringPool(); + List list1 = new ArrayList<>(); + List list2 = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + list1.add("list1-" + i); + list2.add("list2-" + i); + } + Map interned1 = new HashMap<>(); + Map interned2 = new HashMap<>(); + AtomicBoolean failed = new AtomicBoolean(false); + int numThreads = 10; + List threads = new ArrayList<>(); + for (int i = 0; i < numThreads; i++) { + threads.add(new Thread(() -> { + ThreadLocalRandom rand = ThreadLocalRandom.current(); + for (int j = 0; j < 1000000; j++) { + List list; + Map interned; + if (rand.nextBoolean()) { + list = list1; + interned = interned1; + } else { + list = list2; + interned = interned2; + } + // create a new reference + String k = new String(list.get(rand.nextInt(list.size()))); + String v = pool.intern(k); + synchronized (interned) { + String prev = interned.get(k); + if (prev != null) { + // should always return the same reference + if (prev != v) { + failed.set(true); + String msg = "reference not equal, intern failed on string " + k; + LOG.error(msg); + throw new AssertionError(msg); + } + } else { + interned.put(k, v); + } + } + } + })); + } + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + LOG.info("interned1 size {}, interned2 size {}, pool size {}", interned1.size(), + interned2.size(), pool.size()); + assertEquals(interned1.size() + interned2.size(), pool.size()); + interned1.clear(); + list1.clear(); + LOG.info("clear interned1"); + // wait for at most 30 times + for (int i = 0; i < 30; i++) { + // invoke gc manually + LOG.info("trigger GC"); + System.gc(); + Thread.sleep(1000); + // should have cleaned up all the references for list1 + if (interned2.size() == pool.size()) { + return; + } + } + fail("should only have list2 strings in pool, expected pool size " + interned2.size() + + ", but got " + pool.size()); + } +} diff --git a/hbase-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/hbase-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension new file mode 100644 index 000000000000..0cb8a35a1ee8 --- /dev/null +++ b/hbase-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.hadoop.hbase.HBaseJupiterExtension diff --git a/hbase-compression/hbase-compression-lz4/pom.xml b/hbase-compression/hbase-compression-lz4/pom.xml index 240b39c28a11..5a1b9b4aff57 100644 --- a/hbase-compression/hbase-compression-lz4/pom.xml +++ b/hbase-compression/hbase-compression-lz4/pom.xml @@ -86,7 +86,7 @@ - org.lz4 + at.yawk.lz4 lz4-java diff --git a/hbase-endpoint/pom.xml b/hbase-endpoint/pom.xml index f4fdfd4f7fc6..60d2a421b540 100644 --- a/hbase-endpoint/pom.xml +++ b/hbase-endpoint/pom.xml @@ -164,8 +164,18 @@ slf4j-api - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClient.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClient.java index 1126a073fc14..2baf5d493411 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClient.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClient.java @@ -17,13 +17,12 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.LongStream; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.coprocessor.AsyncAggregationClient; @@ -33,19 +32,15 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MediumTests.class, CoprocessorTests.class }) +@Tag(MediumTests.TAG) +@Tag(CoprocessorTests.TAG) public class TestAsyncAggregationClient { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncAggregationClient.class); - private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static TableName TABLE_NAME = TableName.valueOf("TestAsyncAggregationClient"); @@ -62,7 +57,7 @@ public class TestAsyncAggregationClient { private static AsyncTable TABLE; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, @@ -81,7 +76,7 @@ public static void setUp() throws Exception { .collect(Collectors.toList())).get(); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { CONN.close(); UTIL.shutdownMiniCluster(); diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClientPartialResult.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClientPartialResult.java index f199e2163232..3d22d93811c3 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClientPartialResult.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClientPartialResult.java @@ -17,11 +17,10 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.concurrent.ExecutionException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.coprocessor.AsyncAggregationClient; @@ -30,19 +29,15 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MediumTests.class, CoprocessorTests.class }) +@Tag(MediumTests.TAG) +@Tag(CoprocessorTests.TAG) public class TestAsyncAggregationClientPartialResult { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncAggregationClientPartialResult.class); - private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static TableName TABLE_NAME = @@ -56,7 +51,7 @@ public class TestAsyncAggregationClientPartialResult { private static AsyncTable TABLE; - @Before + @BeforeEach public void setUp() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, @@ -71,7 +66,7 @@ public void setUp() throws Exception { TABLE = CONN.getTable(TABLE_NAME); } - @After + @AfterEach public void tearDown() throws Exception { CONN.close(); UTIL.shutdownMiniCluster(); @@ -88,5 +83,4 @@ public void testMin() throws InterruptedException, ExecutionException { assertEquals(2, AsyncAggregationClient .min(TABLE, new LongColumnInterpreter(), new Scan().addColumn(CF, CQ)).get().longValue()); } - } diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClientWithCallbackThreadPool.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClientWithCallbackThreadPool.java index 7b37ddfd1555..3dc23e94dc66 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClientWithCallbackThreadPool.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAggregationClientWithCallbackThreadPool.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -25,7 +25,6 @@ import java.util.stream.Collectors; import java.util.stream.LongStream; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.coprocessor.AsyncAggregationClient; @@ -35,23 +34,19 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Same as TestAsyncAggregationClient, except that {@link AsyncTableImpl} is involved in addition to * {@link RawAsyncTableImpl}. Exercises the code paths in {@link AsyncTableImpl#coprocessorService}. */ -@Category({ MediumTests.class, CoprocessorTests.class }) +@Tag(MediumTests.TAG) +@Tag(CoprocessorTests.TAG) public class TestAsyncAggregationClientWithCallbackThreadPool { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncAggregationClientWithCallbackThreadPool.class); - private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static TableName TABLE_NAME = TableName.valueOf("TestAsyncAggregationClient"); @@ -70,7 +65,7 @@ public class TestAsyncAggregationClientWithCallbackThreadPool { private static ExecutorService EXECUTOR_SERVICE; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, @@ -90,7 +85,7 @@ public static void setUp() throws Exception { .collect(Collectors.toList())).get(); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { CONN.close(); UTIL.shutdownMiniCluster(); @@ -152,5 +147,4 @@ public void testStd() throws InterruptedException, ExecutionException { .std(TABLE, new LongColumnInterpreter(), new Scan().addColumn(CF, CQ)).get().doubleValue(), DELTA); } - } diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java index 5c201dadccaf..3dc5035e32cc 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.List; @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellScannable; import org.apache.hadoop.hbase.CellScanner; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -40,25 +39,20 @@ import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.apache.hbase.thirdparty.com.google.common.collect.ConcurrentHashMultiset; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Multiset; -@Category({ MediumTests.class, ClientTests.class }) +@Tag(MediumTests.TAG) +@Tag(ClientTests.TAG) public class TestRpcControllerFactory { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRpcControllerFactory.class); - public static class StaticRpcControllerFactory extends RpcControllerFactory { public StaticRpcControllerFactory(Configuration conf) { @@ -117,11 +111,8 @@ public void setPriority(TableName tn) { private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - @Rule - public TestName name = new TestName(); - - @BeforeClass - public static void setup() throws Exception { + @BeforeAll + public static void setUp() throws Exception { // load an endpoint so we have an endpoint to test - it doesn't matter which one, but // this is already in tests, so we can just use it. Configuration conf = UTIL.getConfiguration(); @@ -131,8 +122,8 @@ public static void setup() throws Exception { UTIL.startMiniCluster(); } - @AfterClass - public static void teardown() throws Exception { + @AfterAll + public static void tearDown() throws Exception { UTIL.shutdownMiniCluster(); } @@ -143,13 +134,13 @@ public static void teardown() throws Exception { * @throws Exception on failure */ @Test - public void testCountController() throws Exception { + public void testCountController(TestInfo testInfo) throws Exception { Configuration conf = new Configuration(UTIL.getConfiguration()); // setup our custom controller conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY, StaticRpcControllerFactory.class.getName()); - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName()); UTIL.createTable(tableName, fam1).close(); // change one of the connection properties so we get a new Connection with our configuration @@ -246,6 +237,6 @@ public void testFallbackToDefaultRpcControllerFactory() { // Should not fail RpcControllerFactory factory = RpcControllerFactory.instantiate(conf); assertNotNull(factory); - assertEquals(factory.getClass(), RpcControllerFactory.class); + assertEquals(RpcControllerFactory.class, factory.getClass()); } } diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/coprocessor/TestAggregationClient.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/coprocessor/TestAggregationClient.java index 185ef8d2fcf4..6b7c8d0730d1 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/coprocessor/TestAggregationClient.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/coprocessor/TestAggregationClient.java @@ -17,13 +17,12 @@ */ package org.apache.hadoop.hbase.client.coprocessor; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; @@ -35,19 +34,15 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MediumTests.class, CoprocessorTests.class }) +@Tag(MediumTests.TAG) +@Tag(CoprocessorTests.TAG) public class TestAggregationClient { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAggregationClient.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final TableName TABLE_NAME = TableName.valueOf("TestAggregationClient"); @@ -58,7 +53,7 @@ public class TestAggregationClient { private static Table TABLE; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, @@ -69,7 +64,7 @@ public static void setUp() throws Exception { TABLE = CONN.getTable(TABLE_NAME); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { CONN.close(); UTIL.shutdownMiniCluster(); diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateImplementation.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateImplementation.java index 6e83a3f83bf8..51bbb17870c9 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateImplementation.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAggregateImplementation.java @@ -19,10 +19,10 @@ import static org.apache.hadoop.hbase.client.coprocessor.AggregationHelper.getParsedGenericInstance; import static org.apache.hadoop.hbase.quotas.RpcThrottlingException.Type.ReadSizeExceeded; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doAnswer; @@ -37,7 +37,6 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter; @@ -53,23 +52,19 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.stubbing.Answer; /** * Test AggregateImplementation with throttling and partial results */ -@Category({ SmallTests.class, CoprocessorTests.class }) +@Tag(SmallTests.TAG) +@Tag(CoprocessorTests.TAG) public class TestAggregateImplementation { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAggregateImplementation.class); - private static final byte[] CF = Bytes.toBytes("CF"); private static final byte[] CQ = Bytes.toBytes("CQ"); private static final int NUM_ROWS = 5; @@ -85,7 +80,7 @@ public class TestAggregateImplementation { private AggregateRequest request; private RpcController controller; - @Before + @BeforeEach public void setUp() throws Exception { env = mock(RegionCoprocessorEnvironment.class); region = mock(HRegion.class); @@ -172,8 +167,8 @@ public void testMaxWithThrottling() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("Wait interval should be set", 1000, response.getWaitIntervalMs()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(1000, response.getWaitIntervalMs(), "Wait interval should be set"); ByteString b = response.getFirstPart(0); HBaseProtos.LongMsg q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); assertEquals(1L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); @@ -194,11 +189,11 @@ public void testMaxWithThrottling() throws Exception { AggregateResponse response2 = responseCaptor.getValue(); b = response2.getFirstPart(0); q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); - assertEquals("Final max value should be correct", 5L, - (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); + assertEquals(5L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q), + "Final max value should be correct"); - assertFalse("Response should not indicate there are more rows", - response2.hasNextChunkStartRow()); + assertFalse(response2.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); } @Test @@ -222,9 +217,9 @@ public void testMaxThrottleWithNoResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("response should contain the same start row as the request", - request.getScan().getStartRow(), response.getNextChunkStartRow()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(request.getScan().getStartRow(), response.getNextChunkStartRow(), + "response should contain the same start row as the request"); } @Test @@ -266,8 +261,8 @@ public void testMaxDoesNotSupportPartialResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertFalse("Response should not indicate there are more rows", - response.hasNextChunkStartRow()); + assertFalse(response.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); ByteString b = response.getFirstPart(0); HBaseProtos.LongMsg q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); assertEquals(5L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); @@ -285,8 +280,8 @@ public void testMinWithThrottling() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("Wait interval should be set", 1000, response.getWaitIntervalMs()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(1000, response.getWaitIntervalMs(), "Wait interval should be set"); ByteString b = response.getFirstPart(0); HBaseProtos.LongMsg q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); assertEquals(1L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); @@ -307,8 +302,8 @@ public void testMinWithThrottling() throws Exception { b = response.getFirstPart(0); q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); assertEquals(1L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); - assertFalse("Response should not indicate there are more rows", - response2.hasNextChunkStartRow()); + assertFalse(response2.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); } @Test @@ -332,9 +327,9 @@ public void testMinThrottleWithNoResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("response should contain the same start row as the request", - request.getScan().getStartRow(), response.getNextChunkStartRow()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(request.getScan().getStartRow(), response.getNextChunkStartRow(), + "response should contain the same start row as the request"); } @Test @@ -376,8 +371,8 @@ public void testMinDoesNotSupportPartialResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertFalse("Response should not indicate there are more rows", - response.hasNextChunkStartRow()); + assertFalse(response.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); ByteString b = response.getFirstPart(0); HBaseProtos.LongMsg q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); assertEquals(1L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); @@ -394,8 +389,8 @@ public void testSumWithThrottling() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("Wait interval should be set", 1000, response.getWaitIntervalMs()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(1000, response.getWaitIntervalMs(), "Wait interval should be set"); ByteString b = response.getFirstPart(0); HBaseProtos.LongMsg q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); assertEquals(1L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); @@ -417,8 +412,8 @@ public void testSumWithThrottling() throws Exception { b = response2.getFirstPart(0); q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); assertEquals(14L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); - assertFalse("Response should not indicate there are more rows", - response2.hasNextChunkStartRow()); + assertFalse(response2.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); } @Test @@ -442,9 +437,9 @@ public void testSumThrottleWithNoResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("response should contain the same start row as the request", - request.getScan().getStartRow(), response.getNextChunkStartRow()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(request.getScan().getStartRow(), response.getNextChunkStartRow(), + "response should contain the same start row as the request"); } @Test @@ -486,8 +481,8 @@ public void testSumDoesNotSupportPartialResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertFalse("Response should not indicate there are more rows", - response.hasNextChunkStartRow()); + assertFalse(response.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); ByteString b = response.getFirstPart(0); HBaseProtos.LongMsg q = getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, b); assertEquals(15L, (long) LONG_COLUMN_INTERPRETER.getCellValueFromProto(q)); @@ -505,8 +500,8 @@ public void testRowNumWithThrottling() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("Wait interval should be set", 1000, response.getWaitIntervalMs()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(1000, response.getWaitIntervalMs(), "Wait interval should be set"); assertEquals(THROTTLE_AT_ROW - 1, response.getFirstPart(0).asReadOnlyByteBuffer().getLong()); AggregateRequest request2 = AggregateRequest.newBuilder(request) @@ -522,10 +517,11 @@ public void testRowNumWithThrottling() throws Exception { verify(callback2).run(responseCaptor.capture()); AggregateResponse response2 = responseCaptor.getValue(); - assertEquals("Final row count should be correct", NUM_ROWS - THROTTLE_AT_ROW + 1, - response2.getFirstPart(0).asReadOnlyByteBuffer().getLong()); - assertFalse("Response should not indicate there are more rows", - response2.hasNextChunkStartRow()); + assertEquals(NUM_ROWS - THROTTLE_AT_ROW + 1, + response2.getFirstPart(0).asReadOnlyByteBuffer().getLong(), + "Final row count should be correct"); + assertFalse(response2.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); } @Test @@ -544,8 +540,8 @@ public void testRowNumWithScannerCaching() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertFalse("Response should not indicate there are more rows", - response.hasNextChunkStartRow()); + assertFalse(response.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); assertEquals(NUM_ROWS, response.getFirstPart(0).asReadOnlyByteBuffer().getLong()); } @@ -570,9 +566,9 @@ public void testRowNumThrottleWithNoResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("response should contain the same start row as the request", - request.getScan().getStartRow(), response.getNextChunkStartRow()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(request.getScan().getStartRow(), response.getNextChunkStartRow(), + "response should contain the same start row as the request"); } @Test @@ -591,7 +587,7 @@ public void testRowNumWithNoResults() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertFalse("Response should indicate there are no more rows", response.hasNextChunkStartRow()); + assertFalse(response.hasNextChunkStartRow(), "Response should indicate there are no more rows"); assertEquals(0, response.getFirstPart(0).asReadOnlyByteBuffer().getLong()); } @@ -616,10 +612,10 @@ public void testRowNumDoesNotSupportPartialResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertFalse("Response should not indicate there are more rows", - response.hasNextChunkStartRow()); - assertEquals("Final row count should be correct", NUM_ROWS, - response.getFirstPart(0).asReadOnlyByteBuffer().getLong()); + assertFalse(response.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); + assertEquals(NUM_ROWS, response.getFirstPart(0).asReadOnlyByteBuffer().getLong(), + "Final row count should be correct"); } @Test @@ -634,12 +630,14 @@ public void testAvgWithThrottling() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("Wait interval should be set", 1000, response.getWaitIntervalMs()); - assertEquals("sum should be 1", 1L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0)))); - assertEquals("count should be 1", THROTTLE_AT_ROW - 1, - response.getSecondPart().asReadOnlyByteBuffer().getLong()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(1000, response.getWaitIntervalMs(), "Wait interval should be set"); + assertEquals(1L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0))), + "sum should be 1"); + assertEquals(THROTTLE_AT_ROW - 1, response.getSecondPart().asReadOnlyByteBuffer().getLong(), + "count should be 1"); AggregateRequest request2 = AggregateRequest.newBuilder(request) .setScan(request.getScan().toBuilder().setStartRow(response.getNextChunkStartRow()).build()) @@ -654,12 +652,14 @@ public void testAvgWithThrottling() throws Exception { verify(callback2).run(responseCaptor.capture()); AggregateResponse response2 = responseCaptor.getValue(); - assertEquals("sum should be 14", 14L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response2.getFirstPart(0)))); - assertEquals("count should be 4", NUM_ROWS - THROTTLE_AT_ROW + 1, - response2.getSecondPart().asReadOnlyByteBuffer().getLong()); - assertFalse("Response should not indicate there are more rows", - response2.hasNextChunkStartRow()); + assertEquals(14L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response2.getFirstPart(0))), + "sum should be 14"); + assertEquals(NUM_ROWS - THROTTLE_AT_ROW + 1, + response2.getSecondPart().asReadOnlyByteBuffer().getLong(), "count should be 4"); + assertFalse(response2.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); } @Test @@ -683,9 +683,9 @@ public void testAvgThrottleWithNoResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("response should contain the same start row as the request", - request.getScan().getStartRow(), response.getNextChunkStartRow()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(request.getScan().getStartRow(), response.getNextChunkStartRow(), + "response should contain the same start row as the request"); } @Test @@ -727,12 +727,14 @@ public void testAvgDoesNotSupportPartialResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertFalse("Response should not indicate there are more rows", - response.hasNextChunkStartRow()); - assertEquals("sum should be 15", 15L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0)))); - assertEquals("count should be 5", NUM_ROWS, - response.getSecondPart().asReadOnlyByteBuffer().getLong()); + assertFalse(response.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); + assertEquals(15L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0))), + "sum should be 15"); + assertEquals(NUM_ROWS, response.getSecondPart().asReadOnlyByteBuffer().getLong(), + "count should be 5"); } @Test @@ -747,14 +749,18 @@ public void testStdWithThrottling() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("Wait interval should be set", 1000, response.getWaitIntervalMs()); - assertEquals("sum should be 1", 1L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0)))); - assertEquals("sumSq should be 1", 1L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(1)))); - assertEquals("count should be 1", THROTTLE_AT_ROW - 1, - response.getSecondPart().asReadOnlyByteBuffer().getLong()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(1000, response.getWaitIntervalMs(), "Wait interval should be set"); + assertEquals(1L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0))), + "sum should be 1"); + assertEquals(1L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(1))), + "sumSq should be 1"); + assertEquals(THROTTLE_AT_ROW - 1, response.getSecondPart().asReadOnlyByteBuffer().getLong(), + "count should be 1"); AggregateRequest request2 = AggregateRequest.newBuilder(request) .setScan(request.getScan().toBuilder().setStartRow(response.getNextChunkStartRow()).build()) @@ -769,15 +775,18 @@ public void testStdWithThrottling() throws Exception { verify(callback2).run(responseCaptor.capture()); AggregateResponse response2 = responseCaptor.getValue(); - assertEquals("sum should be 14", 14L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response2.getFirstPart(0)))); - assertEquals("sumSq should be 54", 54L, - (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto(getParsedGenericInstance( - LONG_COLUMN_INTERPRETER.getClass(), 3, response2.getFirstPart(1)))); - assertEquals("count should be 4", NUM_ROWS - THROTTLE_AT_ROW + 1, - response2.getSecondPart().asReadOnlyByteBuffer().getLong()); - assertFalse("Response should not indicate there are more rows", - response2.hasNextChunkStartRow()); + assertEquals(14L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response2.getFirstPart(0))), + "sum should be 14"); + assertEquals(54L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response2.getFirstPart(1))), + "sumSq should be 54"); + assertEquals(NUM_ROWS - THROTTLE_AT_ROW + 1, + response2.getSecondPart().asReadOnlyByteBuffer().getLong(), "count should be 4"); + assertFalse(response2.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); } @Test @@ -801,9 +810,9 @@ public void testStdThrottleWithNoResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("response should contain the same start row as the request", - request.getScan().getStartRow(), response.getNextChunkStartRow()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(request.getScan().getStartRow(), response.getNextChunkStartRow(), + "response should contain the same start row as the request"); } @Test @@ -845,15 +854,18 @@ public void testStdDoesNotSupportPartialResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertFalse("Response should not indicate there are more rows", - response.hasNextChunkStartRow()); - assertEquals("sum should be 15", 15L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0)))); - assertEquals("sumSq should be 55", 55L, + assertFalse(response.hasNextChunkStartRow(), + "Response should not indicate there are more rows"); + assertEquals(15L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0))), + "sum should be 15"); + assertEquals(55L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(1)))); - assertEquals("count should be 5", NUM_ROWS, - response.getSecondPart().asReadOnlyByteBuffer().getLong()); + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(1))), + "sumSq should be 55"); + assertEquals(NUM_ROWS, response.getSecondPart().asReadOnlyByteBuffer().getLong(), + "count should be 5"); } @Test @@ -868,10 +880,12 @@ public void testMedianWithThrottling() throws Exception { verify(callback).run(responseCaptor.capture()); AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("Wait interval should be set", 1000, response.getWaitIntervalMs()); - assertEquals("sum should be 1", 1L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0)))); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(1000, response.getWaitIntervalMs(), "Wait interval should be set"); + assertEquals(1L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0))), + "sum should be 1"); AggregateRequest request2 = AggregateRequest.newBuilder(request) .setScan(request.getScan().toBuilder().setStartRow(response.getNextChunkStartRow()).build()) @@ -886,9 +900,11 @@ public void testMedianWithThrottling() throws Exception { verify(callback2).run(responseCaptor.capture()); AggregateResponse response2 = responseCaptor.getValue(); - assertEquals("sum should be 14", 14L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response2.getFirstPart(0)))); - assertFalse("Response should indicate there are more rows", response2.hasNextChunkStartRow()); + assertEquals(14L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response2.getFirstPart(0))), + "sum should be 14"); + assertFalse(response2.hasNextChunkStartRow(), "Response should indicate there are more rows"); } @Test @@ -912,9 +928,9 @@ public void testMedianThrottleWithNoResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertTrue("Response should indicate there are more rows", response.hasNextChunkStartRow()); - assertEquals("response should contain the same start row as the request", - request.getScan().getStartRow(), response.getNextChunkStartRow()); + assertTrue(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); + assertEquals(request.getScan().getStartRow(), response.getNextChunkStartRow(), + "response should contain the same start row as the request"); } @Test @@ -959,9 +975,10 @@ public void testMedianDoesNotSupportPartialResults() throws Exception { AggregateResponse response = responseCaptor.getValue(); - assertEquals("sum should be 15", 15L, (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( - getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0)))); - assertFalse("Response should indicate there are more rows", response.hasNextChunkStartRow()); + assertEquals(15L, + (long) LONG_COLUMN_INTERPRETER.getPromotedValueFromProto( + getParsedGenericInstance(LONG_COLUMN_INTERPRETER.getClass(), 3, response.getFirstPart(0))), + "sum should be 15"); + assertFalse(response.hasNextChunkStartRow(), "Response should indicate there are more rows"); } - } diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAsyncCoprocessorEndpoint.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAsyncCoprocessorEndpoint.java index acecfe2c1462..e4ae0044ce5a 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAsyncCoprocessorEndpoint.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAsyncCoprocessorEndpoint.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.coprocessor; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; @@ -27,10 +27,12 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collections; +import java.util.function.Supplier; import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.AsyncAdmin; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.client.TestAsyncAdminBase; @@ -43,24 +45,23 @@ import org.apache.hadoop.hbase.ipc.protobuf.generated.TestRpcServiceProtos; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@RunWith(Parameterized.class) -@Category({ ClientTests.class, MediumTests.class }) +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestTemplate; + +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}") public class TestAsyncCoprocessorEndpoint extends TestAsyncAdminBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncCoprocessorEndpoint.class); private static final FileNotFoundException WHAT_TO_THROW = new FileNotFoundException("/file.txt"); private static final String DUMMY_VALUE = "val"; - @BeforeClass + public TestAsyncCoprocessorEndpoint(Supplier admin) { + super(admin); + } + + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 60000); TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 120000); @@ -73,7 +74,7 @@ public static void setUpBeforeClass() throws Exception { ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get(); } - @Test + @TestTemplate public void testMasterCoprocessorService() throws Exception { TestProtos.EchoRequestProto request = TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build(); @@ -85,7 +86,7 @@ TestProtos.EchoResponseProto> coprocessorService( assertEquals("hello", response.getMessage()); } - @Test + @TestTemplate public void testMasterCoprocessorError() throws Exception { TestProtos.EmptyRequestProto emptyRequest = TestProtos.EmptyRequestProto.getDefaultInstance(); try { @@ -99,7 +100,7 @@ TestProtos.EmptyResponseProto> coprocessorService( } } - @Test + @TestTemplate public void testRegionServerCoprocessorService() throws Exception { final ServerName serverName = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName(); DummyRegionServerEndpointProtos.DummyRequest request = @@ -113,7 +114,7 @@ DummyRegionServerEndpointProtos.DummyResponse> coprocessorService( assertEquals(DUMMY_VALUE, response.getValue()); } - @Test + @TestTemplate public void testRegionServerCoprocessorServiceError() throws Exception { final ServerName serverName = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName(); DummyRegionServerEndpointProtos.DummyRequest request = diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAsyncCoprocessorOnAllRegionServersEndpoint.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAsyncCoprocessorOnAllRegionServersEndpoint.java index 018c67588029..fd49b2c33a8b 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAsyncCoprocessorOnAllRegionServersEndpoint.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestAsyncCoprocessorOnAllRegionServersEndpoint.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.coprocessor; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; @@ -28,9 +28,11 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import java.util.function.Supplier; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.AsyncAdmin; import org.apache.hadoop.hbase.client.AsyncAdminClientUtils; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.RetriesExhaustedException; @@ -42,27 +44,26 @@ import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestTemplate; -@RunWith(Parameterized.class) -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: admin = {0}") public class TestAsyncCoprocessorOnAllRegionServersEndpoint extends TestAsyncAdminBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncCoprocessorOnAllRegionServersEndpoint.class); private static final String THROW_CLASS_NAME = "java.io.FileNotFoundException"; private static final String DUMMY_VALUE = "val"; private static final int NUM_SLAVES = 5; private static final int NUM_SUCCESS_REGION_SERVERS = 3; - @BeforeClass + public TestAsyncCoprocessorOnAllRegionServersEndpoint(Supplier admin) { + super(admin); + } + + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 60000); TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 120000); @@ -75,12 +76,12 @@ public static void setUpBeforeClass() throws Exception { ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - @Test + @TestTemplate public void testRegionServersCoprocessorService() throws ExecutionException, InterruptedException { DummyRequest request = DummyRequest.getDefaultInstance(); @@ -96,7 +97,7 @@ public void testRegionServersCoprocessorService() }); } - @Test + @TestTemplate public void testRegionServerCoprocessorsServiceAllFail() throws ExecutionException, InterruptedException { DummyRequest request = DummyRequest.getDefaultInstance(); @@ -113,7 +114,7 @@ public void testRegionServerCoprocessorsServiceAllFail() }); } - @Test + @TestTemplate public void testRegionServerCoprocessorsServicePartialFail() throws ExecutionException, InterruptedException { DummyRequest request = DummyRequest.getDefaultInstance(); diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBatchCoprocessorEndpoint.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBatchCoprocessorEndpoint.java index f66fb60da847..a94669d15e18 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBatchCoprocessorEndpoint.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestBatchCoprocessorEndpoint.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.coprocessor; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.protobuf.ByteString; import com.google.protobuf.ServiceException; @@ -26,7 +26,6 @@ import java.util.Map; import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -46,24 +45,20 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * TestEndpoint: test cases to verify the batch execution of coprocessor Endpoint */ -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestBatchCoprocessorEndpoint { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBatchCoprocessorEndpoint.class); - private static final Logger LOG = LoggerFactory.getLogger(TestBatchCoprocessorEndpoint.class); private static final TableName TEST_TABLE = TableName.valueOf("TestTable"); @@ -78,7 +73,7 @@ public class TestBatchCoprocessorEndpoint { private static HBaseTestingUtility util = new HBaseTestingUtility(); - @BeforeClass + @BeforeAll public static void setupBeforeClass() throws Exception { // set configure to indicate which cp should be loaded Configuration conf = util.getConfiguration(); @@ -106,7 +101,7 @@ public static void setupBeforeClass() throws Exception { table.close(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { util.shutdownMiniCluster(); } @@ -136,7 +131,7 @@ public void testAggregationNullResponse() throws Throwable { for (long i = 0; i < rowSeperator2; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); table.close(); } @@ -177,7 +172,7 @@ public void testAggregationWithReturnValue() throws Throwable { for (long i = 0; i < ROWSIZE; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); results.clear(); @@ -193,7 +188,7 @@ public void testAggregationWithReturnValue() throws Throwable { for (long i = rowSeperator1; i < ROWSIZE; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); table.close(); } @@ -212,7 +207,7 @@ public void testAggregation() throws Throwable { for (long i = 0; i < ROWSIZE; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); // scan: for region 2 and region 3 results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1], ROWS[ROWS.length - 1]); @@ -226,7 +221,7 @@ public void testAggregation() throws Throwable { for (long i = rowSeperator1; i < ROWSIZE; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); table.close(); } @@ -272,7 +267,7 @@ public void update(byte[] region, byte[] row, for (long i = 0; i < rowSeperator2; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); assertTrue(hasError); table.close(); } diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java index 60498b237b7f..6833af66e58a 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.coprocessor; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.IOException; @@ -35,7 +35,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -49,26 +48,23 @@ import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.ClassLoaderTestHelper; import org.apache.hadoop.hbase.util.CoprocessorClassLoader; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Test coprocessors class loading. */ -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(LargeTests.TAG) public class TestClassLoading { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestClassLoading.class); private static final Logger LOG = LoggerFactory.getLogger(TestClassLoading.class); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -103,7 +99,7 @@ public Optional getMasterObserver() { new String[] { regionCoprocessor1.getSimpleName(), MultiRowMutationEndpoint.class.getSimpleName(), regionServerCoprocessor.getSimpleName() }; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); @@ -122,7 +118,7 @@ public static void setUpBeforeClass() throws Exception { cluster = TEST_UTIL.getDFSCluster(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } @@ -146,14 +142,14 @@ public void testClassLoadingFromHDFS() throws Exception { new Path(fs.getUri().toString() + Path.SEPARATOR)); String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR + jarFile1.getName(); Path pathOnHDFS1 = new Path(jarFileOnHDFS1); - assertTrue("Copy jar file to HDFS failed.", fs.exists(pathOnHDFS1)); + assertTrue(fs.exists(pathOnHDFS1), "Copy jar file to HDFS failed."); LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1); fs.copyFromLocalFile(new Path(jarFile2.getPath()), new Path(fs.getUri().toString() + Path.SEPARATOR)); String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR + jarFile2.getName(); Path pathOnHDFS2 = new Path(jarFileOnHDFS2); - assertTrue("Copy jar file to HDFS failed.", fs.exists(pathOnHDFS2)); + assertTrue(fs.exists(pathOnHDFS2), "Copy jar file to HDFS failed."); LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2); // create a table that references the coprocessors @@ -206,27 +202,27 @@ public void testClassLoadingFromHDFS() throws Exception { } } - assertTrue("No region was found for table " + tableName, foundTableRegion); - assertTrue("Class " + cpName1 + " was missing on a region", found1); - assertTrue("Class " + cpName2 + " was missing on a region", found2); - assertTrue("Configuration key 'k1' was missing on a region", found2_k1); - assertTrue("Configuration key 'k2' was missing on a region", found2_k2); - assertTrue("Configuration key 'k3' was missing on a region", found2_k3); + assertTrue(foundTableRegion, "No region was found for table " + tableName); + assertTrue(found1, "Class " + cpName1 + " was missing on a region"); + assertTrue(found2, "Class " + cpName2 + " was missing on a region"); + assertTrue(found2_k1, "Configuration key 'k1' was missing on a region"); + assertTrue(found2_k2, "Configuration key 'k2' was missing on a region"); + assertTrue(found2_k3, "Configuration key 'k3' was missing on a region"); // check if CP classloaders are cached - assertNotNull(jarFileOnHDFS1 + " was not cached", - CoprocessorClassLoader.getIfCached(pathOnHDFS1)); - assertNotNull(jarFileOnHDFS2 + " was not cached", - CoprocessorClassLoader.getIfCached(pathOnHDFS2)); + assertNotNull(CoprocessorClassLoader.getIfCached(pathOnHDFS1), + jarFileOnHDFS1 + " was not cached"); + assertNotNull(CoprocessorClassLoader.getIfCached(pathOnHDFS2), + jarFileOnHDFS2 + " was not cached"); // two external jar used, should be one classloader per jar - assertEquals( - "The number of cached classloaders should be equal to the number" + " of external jar files", - 2, CoprocessorClassLoader.getAllCached().size()); + assertEquals(2, CoprocessorClassLoader.getAllCached().size(), + "The number of cached classloaders should be equal to the number" + " of external jar files"); // check if region active classloaders are shared across all RS regions Set externalClassLoaders = new HashSet<>(CoprocessorClassLoader.getAllCached()); for (Map.Entry> regionCP : regionsActiveClassLoaders.entrySet()) { - assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached." - + " ClassLoader Cache:" + externalClassLoaders + " Region ClassLoaders:" - + regionCP.getValue(), externalClassLoaders.containsAll(regionCP.getValue())); + assertTrue(externalClassLoaders.containsAll(regionCP.getValue()), + "Some CP classloaders for region " + regionCP.getKey() + " are not cached." + + " ClassLoader Cache:" + externalClassLoaders + " Region ClassLoaders:" + + regionCP.getValue()); } } @@ -256,7 +252,7 @@ public void testClassLoadingFromLocalFS() throws Exception { found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null); } } - assertTrue("Class " + cpName3 + " was missing on a region", found); + assertTrue(found, "Class " + cpName3 + " was missing on a region"); } @Test @@ -281,12 +277,12 @@ public void testPrivateClassLoader() throws Exception { Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4); if (cp != null) { found = true; - assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader", - cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class); + assertEquals(CoprocessorClassLoader.class, cp.getClass().getClassLoader().getClass(), + "Class " + cpName4 + " was not loaded by CoprocessorClassLoader"); } } } - assertTrue("Class " + cpName4 + " was missing on a region", found); + assertTrue(found, "Class " + cpName4 + " was missing on a region"); } @Test @@ -361,16 +357,16 @@ public void testHBase3810() throws Exception { } } - assertTrue("Class " + cpName1 + " was missing on a region", found_1); - assertTrue("Class " + cpName2 + " was missing on a region", found_2); - assertTrue("Class SimpleRegionObserver was missing on a region", found_3); - assertTrue("Class " + cpName5 + " was missing on a region", found_5); - assertTrue("Class " + cpName6 + " was missing on a region", found_6); + assertTrue(found_1, "Class " + cpName1 + " was missing on a region"); + assertTrue(found_2, "Class " + cpName2 + " was missing on a region"); + assertTrue(found_3, "Class SimpleRegionObserver was missing on a region"); + assertTrue(found_5, "Class " + cpName5 + " was missing on a region"); + assertTrue(found_6, "Class " + cpName6 + " was missing on a region"); - assertTrue("Configuration key 'k1' was missing on a region", found6_k1); - assertTrue("Configuration key 'k2' was missing on a region", found6_k2); - assertTrue("Configuration key 'k3' was missing on a region", found6_k3); - assertFalse("Configuration key 'k4' wasn't configured", found6_k4); + assertTrue(found6_k1, "Configuration key 'k1' was missing on a region"); + assertTrue(found6_k2, "Configuration key 'k2' was missing on a region"); + assertTrue(found6_k3, "Configuration key 'k3' was missing on a region"); + assertFalse(found6_k4, "Configuration key 'k4' wasn't configured"); } @Test @@ -396,7 +392,7 @@ void loadingClassFromLibDirInJar(String libPrefix) throws Exception { fs.copyFromLocalFile(new Path(outerJarFile.getPath()), new Path(fs.getUri().toString() + Path.SEPARATOR)); String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR + outerJarFile.getName(); - assertTrue("Copy jar file to HDFS failed.", fs.exists(new Path(jarFileOnHDFS))); + assertTrue(fs.exists(new Path(jarFileOnHDFS)), "Copy jar file to HDFS failed."); LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS); // create a table that references the coprocessors @@ -438,11 +434,11 @@ void loadingClassFromLibDirInJar(String libPrefix) throws Exception { } } } - assertTrue("Class " + cpName1 + " was missing on a region", found1); - assertTrue("Class " + cpName2 + " was missing on a region", found2); - assertTrue("Configuration key 'k1' was missing on a region", found2_k1); - assertTrue("Configuration key 'k2' was missing on a region", found2_k2); - assertTrue("Configuration key 'k3' was missing on a region", found2_k3); + assertTrue(found1, "Class " + cpName1 + " was missing on a region"); + assertTrue(found2, "Class " + cpName2 + " was missing on a region"); + assertTrue(found2_k1, "Configuration key 'k1' was missing on a region"); + assertTrue(found2_k2, "Configuration key 'k2' was missing on a region"); + assertTrue(found2_k3, "Configuration key 'k3' was missing on a region"); } @Test diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java index 3c64c2099c23..0678c338d150 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpoint.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.hbase.coprocessor; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -31,7 +31,6 @@ import java.util.Map; import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -54,22 +53,19 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * TestEndpoint: test cases to verify coprocessor Endpoint */ -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestCoprocessorEndpoint { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCoprocessorEndpoint.class); private static final Logger LOG = LoggerFactory.getLogger(TestCoprocessorEndpoint.class); @@ -85,7 +81,7 @@ public class TestCoprocessorEndpoint { private static HBaseTestingUtility util = new HBaseTestingUtility(); - @BeforeClass + @BeforeAll public static void setupBeforeClass() throws Exception { // set configure to indicate which cp should be loaded Configuration conf = util.getConfiguration(); @@ -111,7 +107,7 @@ public static void setupBeforeClass() throws Exception { table.close(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { util.shutdownMiniCluster(); } @@ -151,7 +147,7 @@ public void testAggregation() throws Throwable { for (long i = 0; i < ROWSIZE; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); results.clear(); @@ -166,7 +162,7 @@ public void testAggregation() throws Throwable { for (int i = rowSeperator1; i < ROWSIZE; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); table.close(); } diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpointTracing.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpointTracing.java index 007efbf56515..8fff6ea5571b 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpointTracing.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorEndpointTracing.java @@ -30,14 +30,15 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasProperty; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.protobuf.Descriptors; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; import io.opentelemetry.sdk.trace.data.SpanData; import java.util.List; import java.util.Map; @@ -47,13 +48,12 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ConnectionRule; -import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ConnectionExtension; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MatcherPredicate; -import org.apache.hadoop.hbase.MiniClusterRule; +import org.apache.hadoop.hbase.MiniClusterExtension; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Admin; @@ -76,19 +76,16 @@ import org.apache.hadoop.hbase.ipc.protobuf.generated.TestRpcServiceProtos.TestProtobufRpcProto; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.trace.OpenTelemetryClassRule; -import org.apache.hadoop.hbase.trace.OpenTelemetryTestRule; import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.util.Bytes; import org.hamcrest.Matcher; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.ExternalResource; -import org.junit.rules.RuleChain; -import org.junit.rules.TestName; -import org.junit.rules.TestRule; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,18 +94,27 @@ /** * Test cases to verify tracing coprocessor Endpoint execution */ -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestCoprocessorEndpointTracing { + private static final Logger logger = LoggerFactory.getLogger(TestCoprocessorEndpointTracing.class); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCoprocessorEndpointTracing.class); + private static final TableName TEST_TABLE = + TableName.valueOf(TestCoprocessorEndpointTracing.class.getSimpleName()); + private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily"); + + private String testName; + + @Order(1) + @RegisterExtension + private static final OpenTelemetryExtension otelExtension = OpenTelemetryExtension.create(); - private static final OpenTelemetryClassRule otelClassRule = OpenTelemetryClassRule.create(); - private static final MiniClusterRule miniclusterRule = - MiniClusterRule.newBuilder().setConfiguration(() -> { + @Order(2) + @RegisterExtension + private static final MiniClusterExtension miniClusterExtension = + MiniClusterExtension.newBuilder().setConfiguration(() -> { final Configuration conf = HBaseConfiguration.create(); conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 5000); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, @@ -117,39 +123,32 @@ public class TestCoprocessorEndpointTracing { ProtobufCoprocessorService.class.getName()); return conf; }).build(); - private static final ConnectionRule connectionRule = ConnectionRule.createConnectionRule( - miniclusterRule::createConnection, miniclusterRule::createAsyncConnection); - - private static final class Setup extends ExternalResource { - @Override - protected void before() throws Throwable { - final HBaseTestingUtility util = miniclusterRule.getTestingUtility(); - final AsyncConnection connection = connectionRule.getAsyncConnection(); - final AsyncAdmin admin = connection.getAdmin(); - final TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TEST_TABLE) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY)).build(); - admin.createTable(tableDescriptor).get(); - util.waitUntilAllRegionsAssigned(TEST_TABLE); - } - } - @ClassRule - public static final TestRule testRule = RuleChain.outerRule(otelClassRule).around(miniclusterRule) - .around(connectionRule).around(new Setup()); + @Order(3) + @RegisterExtension + private static final ConnectionExtension connectionExtension = + ConnectionExtension.createConnectionExtension(miniClusterExtension::createConnection, + miniClusterExtension::createAsyncConnection); - private static final TableName TEST_TABLE = - TableName.valueOf(TestCoprocessorEndpointTracing.class.getSimpleName()); - private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily"); - - @Rule - public OpenTelemetryTestRule otelTestRule = new OpenTelemetryTestRule(otelClassRule); + @BeforeAll + public static void beforeAll() throws Exception { + final HBaseTestingUtility util = miniClusterExtension.getTestingUtility(); + final AsyncConnection connection = connectionExtension.getAsyncConnection(); + final AsyncAdmin admin = connection.getAdmin(); + final TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TEST_TABLE) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY)).build(); + admin.createTable(tableDescriptor).get(); + util.waitUntilAllRegionsAssigned(TEST_TABLE); + } - @Rule - public TestName testName = new TestName(); + @BeforeEach + public void setUp(TestInfo testInfo) throws Exception { + testName = testInfo.getTestMethod().get().getName(); + } @Test public void traceAsyncTableEndpoint() { - final AsyncConnection connection = connectionRule.getAsyncConnection(); + final AsyncConnection connection = connectionExtension.getAsyncConnection(); final AsyncTable table = connection.getTable(TEST_TABLE); final EchoRequestProto request = EchoRequestProto.newBuilder().setMessage("hello").build(); final CompletableFuture> future = new CompletableFuture<>(); @@ -194,14 +193,14 @@ public void onError(Throwable error) { } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } - }, testName.getMethodName()); + }, testName); assertNotNull(results); - assertTrue("coprocessor call returned no results.", MapUtils.isNotEmpty(results)); + assertTrue(MapUtils.isNotEmpty(results), "coprocessor call returned no results."); assertThat(results.values(), everyItem(allOf(notNullValue(), equalTo("hello")))); - final Matcher parentMatcher = allOf(hasName(testName.getMethodName()), hasEnded()); + final Matcher parentMatcher = allOf(hasName(testName), hasEnded()); waitForAndLog(parentMatcher); - final List spans = otelClassRule.getSpans(); + final List spans = otelExtension.getSpans(); final SpanData testSpan = spans.stream().filter(parentMatcher::matches).findFirst().orElseThrow(AssertionError::new); @@ -217,7 +216,7 @@ public void onError(Throwable error) { @Test public void traceSyncTableEndpointCall() throws Exception { - final Connection connection = connectionRule.getConnection(); + final Connection connection = connectionExtension.getConnection(); try (final Table table = connection.getTable(TEST_TABLE)) { final RpcController controller = new ServerRpcController(); final EchoRequestProto request = EchoRequestProto.newBuilder().setMessage("hello").build(); @@ -232,16 +231,16 @@ public void traceSyncTableEndpointCall() throws Exception { } catch (Throwable t) { throw new RuntimeException(t); } - }, testName.getMethodName()); + }, testName); assertNotNull(results); - assertTrue("coprocessor call returned no results.", MapUtils.isNotEmpty(results)); + assertTrue(MapUtils.isNotEmpty(results), "coprocessor call returned no results."); assertThat(results.values(), everyItem(allOf(notNullValue(), hasProperty("message", equalTo("hello"))))); } - final Matcher parentMatcher = allOf(hasName(testName.getMethodName()), hasEnded()); + final Matcher parentMatcher = allOf(hasName(testName), hasEnded()); waitForAndLog(parentMatcher); - final List spans = otelClassRule.getSpans(); + final List spans = otelExtension.getSpans(); final SpanData testSpan = spans.stream().filter(parentMatcher::matches).findFirst().orElseThrow(AssertionError::new); @@ -257,7 +256,7 @@ public void traceSyncTableEndpointCall() throws Exception { @Test public void traceSyncTableEndpointCallAndCallback() throws Exception { - final Connection connection = connectionRule.getConnection(); + final Connection connection = connectionExtension.getConnection(); try (final Table table = connection.getTable(TEST_TABLE)) { final RpcController controller = new ServerRpcController(); final EchoRequestProto request = EchoRequestProto.newBuilder().setMessage("hello").build(); @@ -273,16 +272,16 @@ public void traceSyncTableEndpointCallAndCallback() throws Exception { } catch (Throwable t) { throw new RuntimeException(t); } - }, testName.getMethodName()); + }, testName); assertNotNull(results); - assertTrue("coprocessor call returned no results.", MapUtils.isNotEmpty(results)); + assertTrue(MapUtils.isNotEmpty(results), "coprocessor call returned no results."); assertThat(results.values(), everyItem(allOf(notNullValue(), hasProperty("message", equalTo("hello"))))); } - final Matcher parentMatcher = allOf(hasName(testName.getMethodName()), hasEnded()); + final Matcher parentMatcher = allOf(hasName(testName), hasEnded()); waitForAndLog(parentMatcher); - final List spans = otelClassRule.getSpans(); + final List spans = otelExtension.getSpans(); final SpanData testSpan = spans.stream().filter(parentMatcher::matches).findFirst().orElseThrow(AssertionError::new); @@ -298,7 +297,7 @@ public void traceSyncTableEndpointCallAndCallback() throws Exception { @Test public void traceSyncTableRegionCoprocessorRpcChannel() throws Exception { - final Connection connection = connectionRule.getConnection(); + final Connection connection = connectionExtension.getConnection(); try (final Table table = connection.getTable(TEST_TABLE)) { final EchoRequestProto request = EchoRequestProto.newBuilder().setMessage("hello").build(); final EchoResponseProto response = TraceUtil.trace(() -> { @@ -310,14 +309,14 @@ public void traceSyncTableRegionCoprocessorRpcChannel() throws Exception { } catch (Throwable t) { throw new RuntimeException(t); } - }, testName.getMethodName()); + }, testName); assertNotNull(response); assertEquals("hello", response.getMessage()); } - final Matcher parentMatcher = allOf(hasName(testName.getMethodName()), hasEnded()); + final Matcher parentMatcher = allOf(hasName(testName), hasEnded()); waitForAndLog(parentMatcher); - final List spans = otelClassRule.getSpans(); + final List spans = otelExtension.getSpans(); /* * This interface is really low level: it returns a Channel and expects the caller to invoke it. @@ -333,7 +332,7 @@ public void traceSyncTableRegionCoprocessorRpcChannel() throws Exception { @Test public void traceSyncTableBatchEndpoint() throws Exception { - final Connection connection = connectionRule.getConnection(); + final Connection connection = connectionExtension.getConnection(); try (final Table table = connection.getTable(TEST_TABLE)) { final Descriptors.MethodDescriptor descriptor = TestProtobufRpcProto.getDescriptor().findMethodByName("echo"); @@ -345,15 +344,15 @@ public void traceSyncTableBatchEndpoint() throws Exception { } catch (Throwable t) { throw new RuntimeException(t); } - }, testName.getMethodName()); + }, testName); assertNotNull(response); assertThat(response.values(), everyItem(allOf(notNullValue(), hasProperty("message", equalTo("hello"))))); } - final Matcher parentMatcher = allOf(hasName(testName.getMethodName()), hasEnded()); + final Matcher parentMatcher = allOf(hasName(testName), hasEnded()); waitForAndLog(parentMatcher); - final List spans = otelClassRule.getSpans(); + final List spans = otelExtension.getSpans(); final SpanData testSpan = spans.stream().filter(parentMatcher::matches).findFirst().orElseThrow(AssertionError::new); @@ -369,7 +368,7 @@ public void traceSyncTableBatchEndpoint() throws Exception { @Test public void traceSyncTableBatchEndpointCallback() throws Exception { - final Connection connection = connectionRule.getConnection(); + final Connection connection = connectionExtension.getConnection(); try (final Table table = connection.getTable(TEST_TABLE)) { final Descriptors.MethodDescriptor descriptor = TestProtobufRpcProto.getDescriptor().findMethodByName("echo"); @@ -382,16 +381,16 @@ public void traceSyncTableBatchEndpointCallback() throws Exception { } catch (Throwable t) { throw new RuntimeException(t); } - }, testName.getMethodName()); + }, testName); assertNotNull(results); - assertTrue("coprocessor call returned no results.", MapUtils.isNotEmpty(results)); + assertTrue(MapUtils.isNotEmpty(results), "coprocessor call returned no results."); assertThat(results.values(), everyItem(allOf(notNullValue(), hasProperty("message", equalTo("hello"))))); } - final Matcher parentMatcher = allOf(hasName(testName.getMethodName()), hasEnded()); + final Matcher parentMatcher = allOf(hasName(testName), hasEnded()); waitForAndLog(parentMatcher); - final List spans = otelClassRule.getSpans(); + final List spans = otelExtension.getSpans(); final SpanData testSpan = spans.stream().filter(parentMatcher::matches).findFirst().orElseThrow(AssertionError::new); @@ -407,7 +406,7 @@ public void traceSyncTableBatchEndpointCallback() throws Exception { @Test public void traceAsyncAdminEndpoint() throws Exception { - final AsyncConnection connection = connectionRule.getAsyncConnection(); + final AsyncConnection connection = connectionExtension.getAsyncConnection(); final AsyncAdmin admin = connection.getAdmin(); final EchoRequestProto request = EchoRequestProto.newBuilder().setMessage("hello").build(); final ServiceCaller callback = @@ -415,13 +414,13 @@ public void traceAsyncAdminEndpoint() throws Exception { final String response = TraceUtil .tracedFuture(() -> admin.coprocessorService(TestProtobufRpcProto::newStub, callback), - testName.getMethodName()) + testName) .get().getMessage(); assertEquals("hello", response); - final Matcher parentMatcher = allOf(hasName(testName.getMethodName()), hasEnded()); + final Matcher parentMatcher = allOf(hasName(testName), hasEnded()); waitForAndLog(parentMatcher); - final List spans = otelClassRule.getSpans(); + final List spans = otelExtension.getSpans(); final SpanData testSpan = spans.stream().filter(parentMatcher::matches).findFirst().orElseThrow(AssertionError::new); @@ -432,7 +431,7 @@ public void traceAsyncAdminEndpoint() throws Exception { @Test public void traceSyncAdminEndpoint() throws Exception { - final Connection connection = connectionRule.getConnection(); + final Connection connection = connectionExtension.getConnection(); try (final Admin admin = connection.getAdmin()) { final TestProtobufRpcProto.BlockingInterface service = TestProtobufRpcProto.newBlockingStub(admin.coprocessorService()); @@ -443,13 +442,13 @@ public void traceSyncAdminEndpoint() throws Exception { } catch (ServiceException e) { throw new RuntimeException(e); } - }, testName.getMethodName()); + }, testName); assertEquals("hello", response); } - final Matcher parentMatcher = allOf(hasName(testName.getMethodName()), hasEnded()); + final Matcher parentMatcher = allOf(hasName(testName), hasEnded()); waitForAndLog(parentMatcher); - final List spans = otelClassRule.getSpans(); + final List spans = otelExtension.getSpans(); final SpanData testSpan = spans.stream().filter(parentMatcher::matches).findFirst().orElseThrow(AssertionError::new); @@ -459,10 +458,10 @@ public void traceSyncAdminEndpoint() throws Exception { } private void waitForAndLog(Matcher spanMatcher) { - final Configuration conf = connectionRule.getAsyncConnection().getConfiguration(); + final Configuration conf = connectionExtension.getAsyncConnection().getConfiguration(); Waiter.waitFor(conf, TimeUnit.SECONDS.toMillis(5), - new MatcherPredicate<>(otelClassRule::getSpans, hasItem(spanMatcher))); - final List spans = otelClassRule.getSpans(); + new MatcherPredicate<>(otelExtension::getSpans, hasItem(spanMatcher))); + final List spans = otelExtension.getSpans(); if (logger.isDebugEnabled()) { StringTraceRenderer renderer = new StringTraceRenderer(spans); renderer.render(logger::debug); diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorTableEndpoint.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorTableEndpoint.java index ceb5bcf3bcce..1be250ce2b72 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorTableEndpoint.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorTableEndpoint.java @@ -17,13 +17,12 @@ */ package org.apache.hadoop.hbase.coprocessor; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.protobuf.ByteString; import com.google.protobuf.ServiceException; import java.io.IOException; import java.util.Map; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -37,19 +36,16 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; - -@Category({ CoprocessorTests.class, MediumTests.class }) +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; + +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestCoprocessorTableEndpoint { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCoprocessorTableEndpoint.class); private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily"); private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier"); @@ -61,22 +57,26 @@ public class TestCoprocessorTableEndpoint { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - @Rule - public TestName name = new TestName(); + private String methodName; - @BeforeClass + @BeforeAll public static void setupBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(2); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } + @BeforeEach + public void setUpEach(TestInfo testInfo) { + this.methodName = testInfo.getTestMethod().get().getName(); + } + @Test public void testCoprocessorTableEndpoint() throws Throwable { - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = TableName.valueOf(methodName); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(TEST_FAMILY)); @@ -88,7 +88,7 @@ public void testCoprocessorTableEndpoint() throws Throwable { @Test public void testDynamicCoprocessorTableEndpoint() throws Throwable { - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = TableName.valueOf(methodName); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(TEST_FAMILY)); @@ -167,7 +167,7 @@ private static final void verifyTable(TableName tableName) throws Throwable { for (long i = 0; i < ROWSIZE; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); // scan: for region 2 and region 3 results.clear(); @@ -180,7 +180,7 @@ private static final void verifyTable(TableName tableName) throws Throwable { for (int i = rowSeperator1; i < ROWSIZE; i++) { expectedResult += i; } - assertEquals("Invalid result", expectedResult, sumResult); + assertEquals(expectedResult, sumResult, "Invalid result"); } finally { table.close(); } diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestImportExport.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestImportExport.java index fa025e120c1f..3fc90ad703d0 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestImportExport.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestImportExport.java @@ -18,26 +18,27 @@ package org.apache.hadoop.hbase.coprocessor; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.mapreduce.TestImportExportBase; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MediumTests.class }) -public class TestImportExport extends org.apache.hadoop.hbase.mapreduce.TestImportExport { +@Tag(LargeTests.TAG) +public class TestImportExport extends TestImportExportBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestImportExport.class); - - @BeforeClass + @BeforeAll public static void beforeClass() throws Throwable { UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, org.apache.hadoop.hbase.coprocessor.Export.class.getName()); - org.apache.hadoop.hbase.mapreduce.TestImportExport.beforeClass(); + TestImportExportBase.setUpBeforeClass(); + } + + @AfterAll + public static void afterClass() throws Throwable { + UTIL.shutdownMiniCluster(); } @Override @@ -52,7 +53,7 @@ protected void runExportMain(String[] args) throws Throwable { } @Test - @Ignore + @Disabled @Override public void testImport94Table() throws Throwable { // Skip the test which is unrelated to the coprocessor.Export. diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorEndpoint.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorEndpoint.java index d9a7aec4670e..9fbb9b253b7f 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorEndpoint.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorEndpoint.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hbase.coprocessor; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; @@ -26,7 +27,6 @@ import java.io.FileNotFoundException; import java.util.Collections; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos; @@ -38,26 +38,22 @@ import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(LargeTests.TAG) public class TestRegionServerCoprocessorEndpoint { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionServerCoprocessorEndpoint.class); - public static final FileNotFoundException WHAT_TO_THROW = new FileNotFoundException("/file.txt"); private static HBaseTestingUtility TEST_UTIL = null; private static Configuration CONF = null; private static final String DUMMY_VALUE = "val"; - @BeforeClass + @BeforeAll public static void setupBeforeClass() throws Exception { TEST_UTIL = new HBaseTestingUtility(); CONF = TEST_UTIL.getConfiguration(); @@ -66,7 +62,7 @@ public static void setupBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } @@ -101,7 +97,7 @@ public void testEndpointExceptions() throws Exception { TEST_UTIL.getAdmin().coprocessorService(serverName)); service.dummyThrow(controller, DummyRegionServerEndpointProtos.DummyRequest.getDefaultInstance(), rpcCallback); - assertEquals(null, rpcCallback.get()); + assertNull(rpcCallback.get()); assertTrue(controller.failedOnException()); assertEquals(WHAT_TO_THROW.getClass().getName().trim(), ((RemoteWithExtrasException) controller.getFailedOn().getCause()).getClassName().trim()); diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestSecureExport.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestSecureExport.java index fae6edd4970b..bad387a36f31 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestSecureExport.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestSecureExport.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.coprocessor; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.protobuf.ServiceException; import java.io.File; @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; @@ -62,32 +61,27 @@ import org.apache.hadoop.hbase.security.visibility.VisibilityClient; import org.apache.hadoop.hbase.security.visibility.VisibilityConstants; import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; -@Category({ MediumTests.class }) +@Tag(LargeTests.TAG) public class TestSecureExport { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSecureExport.class); private static final Logger LOG = LoggerFactory.getLogger(TestSecureExport.class); private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); @@ -122,8 +116,7 @@ public class TestSecureExport { private static final String CONFIDENTIAL = "confidential"; private static final String SECRET = "secret"; private static final String TOPSECRET = "topsecret"; - @Rule - public final TestName name = new TestName(); + private String methodName; private static void setUpKdcServer() throws Exception { KDC = UTIL.setupMiniKdc(KEYTAB_FILE); @@ -170,12 +163,13 @@ private static void addLabels(final Configuration conf, final List users getUserByLogin(USER_ADMIN).runAs(action); } - @Before - public void announce() { - LOG.info("Running " + name.getMethodName()); + @BeforeEach + public void announce(TestInfo testInfo) { + methodName = testInfo.getTestMethod().get().getName(); + LOG.info("Running " + methodName); } - @After + @AfterEach public void cleanup() throws IOException { } @@ -189,7 +183,7 @@ private static void clearOutput(Path path) throws IOException { /** * Sets the security firstly for getting the correct default realm. */ - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { UserProvider.setUserProviderForTesting(UTIL.getConfiguration(), HadoopSecurityEnabledUserProviderForTesting.class); @@ -210,7 +204,7 @@ public static void beforeClass() throws Exception { Arrays.asList(PRIVATE, CONFIDENTIAL, SECRET, TOPSECRET)); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { if (KDC != null) { KDC.stop(); @@ -224,11 +218,10 @@ public static void afterClass() throws Exception { */ @Test public void testAccessCase() throws Throwable { - final String exportTable = name.getMethodName(); - TableDescriptor exportHtd = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILYA)).setOwnerString(USER_OWNER) - .build(); + final String exportTable = methodName; + TableDescriptor exportHtd = TableDescriptorBuilder.newBuilder(TableName.valueOf(exportTable)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILYA)).setOwnerString(USER_OWNER) + .build(); SecureTestUtil.createTable(UTIL, exportHtd, new byte[][] { Bytes.toBytes("s") }); SecureTestUtil.grantOnTable(UTIL, USER_RO, TableName.valueOf(exportTable), null, null, Permission.Action.READ); @@ -283,11 +276,11 @@ public void testAccessCase() throws Throwable { // owned by the current user, not the hbase service user. FileStatus outputDirFileStatus = fs.getFileStatus(new Path(openDir, "output")); String currentUserName = User.getCurrent().getShortName(); - assertEquals("Unexpected file owner", currentUserName, outputDirFileStatus.getOwner()); + assertEquals(currentUserName, outputDirFileStatus.getOwner(), "Unexpected file owner"); FileStatus[] outputFileStatus = fs.listStatus(new Path(openDir, "output")); for (FileStatus fileStatus : outputFileStatus) { - assertEquals("Unexpected file owner", currentUserName, fileStatus.getOwner()); + assertEquals(currentUserName, fileStatus.getOwner(), "Unexpected file owner"); } } else { LOG.info("output directory doesn't exist. Skip check"); @@ -311,8 +304,8 @@ public void testAccessCase() throws Throwable { @Test @org.junit.Ignore // See HBASE-23990 public void testVisibilityLabels() throws IOException, Throwable { - final String exportTable = name.getMethodName() + "_export"; - final String importTable = name.getMethodName() + "_import"; + final String exportTable = methodName + "_export"; + final String importTable = methodName + "_import"; final TableDescriptor exportHtd = TableDescriptorBuilder.newBuilder(TableName.valueOf(exportTable)) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILYA)).setOwnerString(USER_OWNER) diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/ipc/TestCoprocessorRpcUtils.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/ipc/TestCoprocessorRpcUtils.java index 0d75f6e625a4..29888b2deab1 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/ipc/TestCoprocessorRpcUtils.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/ipc/TestCoprocessorRpcUtils.java @@ -17,24 +17,18 @@ */ package org.apache.hadoop.hbase.ipc; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.protobuf.Descriptors; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos; import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestCoprocessorRpcUtils { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCoprocessorRpcUtils.class); - @Test public void testServiceName() throws Exception { // verify that we de-namespace build in HBase rpc services diff --git a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java index 6cfeaf89a5fa..9cab130ed266 100644 --- a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java +++ b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; @@ -29,7 +29,6 @@ import java.util.Collections; import java.util.Map; import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; @@ -56,21 +55,18 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ RegionServerTests.class, MediumTests.class }) +@Tag(RegionServerTests.TAG) +@Tag(MediumTests.TAG) public class TestServerCustomProtocol { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestServerCustomProtocol.class); private static final Logger LOG = LoggerFactory.getLogger(TestServerCustomProtocol.class); static final String WHOAREYOU = "Who are you?"; @@ -150,14 +146,14 @@ public Iterable getServices() { private static HBaseTestingUtility util = new HBaseTestingUtility(); - @BeforeClass + @BeforeAll public static void setupBeforeClass() throws Exception { util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, PingHandler.class.getName()); util.startMiniCluster(); } - @Before + @BeforeEach public void before() throws Exception { final byte[][] SPLIT_KEYS = new byte[][] { ROW_B, ROW_C }; Table table = util.createTable(TEST_TABLE, TEST_FAMILY, SPLIT_KEYS); @@ -175,12 +171,12 @@ public void before() throws Exception { table.put(putc); } - @After + @AfterEach public void after() throws Exception { util.deleteTable(TEST_TABLE); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { util.shutdownMiniCluster(); } @@ -192,7 +188,7 @@ public void testSingleProxy() throws Throwable { // There are three regions so should get back three results. assertEquals(3, results.size()); for (Map.Entry e : results.entrySet()) { - assertEquals("Invalid custom protocol response", "pong", e.getValue()); + assertEquals("pong", e.getValue(), "Invalid custom protocol response"); } hello(table, "George", HELLO + "George"); LOG.info("Did george"); @@ -239,7 +235,7 @@ private Map hello(final Table table, final String send, final St throws ServiceException, Throwable { Map results = hello(table, send); for (Map.Entry e : results.entrySet()) { - assertEquals("Invalid custom protocol response", response, e.getValue()); + assertEquals(response, e.getValue(), "Invalid custom protocol response"); } return results; } @@ -355,8 +351,8 @@ public void testRowRange() throws Throwable { assertEquals(2, results.size()); // should contain last 2 regions HRegionLocation loc = locator.getRegionLocation(ROW_A, true); - assertNull("Should be missing region for row aaa (prior to start row)", - results.get(loc.getRegionInfo().getRegionName())); + assertNull(results.get(loc.getRegion().getRegionName()), + "Should be missing region for row aaa (prior to start row)"); verifyRegionResults(locator, results, ROW_B); verifyRegionResults(locator, results, ROW_C); @@ -367,8 +363,8 @@ public void testRowRange() throws Throwable { verifyRegionResults(locator, results, ROW_A); verifyRegionResults(locator, results, ROW_B); loc = locator.getRegionLocation(ROW_C, true); - assertNull("Should be missing region for row ccc (past stop row)", - results.get(loc.getRegionInfo().getRegionName())); + assertNull(results.get(loc.getRegion().getRegionName()), + "Should be missing region for row ccc (past stop row)"); // test explicit start + end results = ping(table, ROW_AB, ROW_BC); @@ -377,8 +373,8 @@ public void testRowRange() throws Throwable { verifyRegionResults(locator, results, ROW_A); verifyRegionResults(locator, results, ROW_B); loc = locator.getRegionLocation(ROW_C, true); - assertNull("Should be missing region for row ccc (past stop row)", - results.get(loc.getRegionInfo().getRegionName())); + assertNull(results.get(loc.getRegion().getRegionName()), + "Should be missing region for row ccc (past stop row)"); // test single region results = ping(table, ROW_B, ROW_BC); @@ -386,11 +382,11 @@ public void testRowRange() throws Throwable { assertEquals(1, results.size()); verifyRegionResults(locator, results, ROW_B); loc = locator.getRegionLocation(ROW_A, true); - assertNull("Should be missing region for row aaa (prior to start)", - results.get(loc.getRegionInfo().getRegionName())); + assertNull(results.get(loc.getRegion().getRegionName()), + "Should be missing region for row aaa (prior to start)"); loc = locator.getRegionLocation(ROW_C, true); - assertNull("Should be missing region for row ccc (past stop row)", - results.get(loc.getRegionInfo().getRegionName())); + assertNull(results.get(loc.getRegion().getRegionName()), + "Should be missing region for row ccc (past stop row)"); } } @@ -449,7 +445,7 @@ public void testNullReturn() throws Throwable { public void testEmptyReturnType() throws Throwable { try (Table table = util.getConnection().getTable(TEST_TABLE)) { Map results = noop(table, ROW_A, ROW_C); - assertEquals("Should have results from three regions", 3, results.size()); + assertEquals(3, results.size(), "Should have results from three regions"); // all results should be null for (Object v : results.values()) { assertNull(v); @@ -470,9 +466,9 @@ private void verifyRegionResults(RegionLocator regionLocator, Map }). - * Python: hbase-examples/src/main/python/DemoClient.py - 1. Modify the added system path in the file to point to {$THRIFT_HOME}/lib/py/build/lib.[YOUR SYSTEM] + * Python: hbase-examples/src/main/python/thrift1/DemoClient.py and hbase-examples/src/main/python/thrift2/DemoClient.py + 1. Install the thrift package to your local Python environment: pip3 install thrift 2. Execute {python DemoClient.py }. * PHP: hbase-examples/src/main/php/DemoClient.php diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml index aa84fdf3ceb7..24207cb24b23 100644 --- a/hbase-examples/pom.xml +++ b/hbase-examples/pom.xml @@ -134,8 +134,18 @@ hbase-rest - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/thrift/DemoClient.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/thrift/DemoClient.java index 62b2cbecd7b1..0e5b431d8b3c 100644 --- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/thrift/DemoClient.java +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/thrift/DemoClient.java @@ -144,9 +144,8 @@ private void run() throws Exception { } // Create the demo table with two column families, entry: and unused: - ArrayList columns = new ArrayList<>(2); - ColumnDescriptor col; - col = new ColumnDescriptor(); + List columns = new ArrayList<>(2); + ColumnDescriptor col = new ColumnDescriptor(); col.name = ByteBuffer.wrap(bytes("entry:")); col.timeToLive = Integer.MAX_VALUE; col.maxVersions = 10; @@ -157,7 +156,6 @@ private void run() throws Exception { columns.add(col); System.out.println("creating table: " + ClientUtils.utf8(demoTable.array())); - try { client.createTable(demoTable, columns); client.createTable(disabledTable, columns); @@ -167,7 +165,6 @@ private void run() throws Exception { System.out.println("column families in " + ClientUtils.utf8(demoTable.array()) + ": "); Map columnMap = client.getColumnDescriptors(demoTable); - for (ColumnDescriptor col2 : columnMap.values()) { System.out.println( " column: " + ClientUtils.utf8(col2.name.array()) + ", maxVer: " + col2.maxVersions); @@ -215,8 +212,8 @@ private void run() throws Exception { client.mutateRow(demoTable, ByteBuffer.wrap(invalid), mutations, dummyAttributes); // Run a scanner on the rows we just created - ArrayList columnNames = new ArrayList<>(); - columnNames.add(ByteBuffer.wrap(bytes("entry:"))); + List columnNames = new ArrayList<>(); + columnNames.add(ByteBuffer.wrap(bytes("entry"))); System.out.println("Starting scanner..."); int scanner = @@ -231,6 +228,7 @@ private void run() throws Exception { printRow(entry); } + System.out.println("Scanner finished..."); // Run some operations on a bunch of rows for (int i = 100; i >= 0; --i) { @@ -262,9 +260,8 @@ private void run() throws Exception { client.mutateRow(demoTable, ByteBuffer.wrap(row), mutations, dummyAttributes); printRow(client.getRow(demoTable, ByteBuffer.wrap(row), dummyAttributes)); - Mutation m; mutations = new ArrayList<>(2); - m = new Mutation(); + Mutation m = new Mutation(); m.column = ByteBuffer.wrap(bytes("entry:foo")); m.isDelete = true; mutations.add(m); @@ -319,7 +316,7 @@ private void run() throws Exception { System.exit(-1); } - System.out.println(""); + System.out.println(); } // scan all rows/columnNames @@ -327,9 +324,10 @@ private void run() throws Exception { for (ColumnDescriptor col2 : client.getColumnDescriptors(demoTable).values()) { System.out.println("column with name: " + ClientUtils.utf8(col2.name)); - System.out.println(col2.toString()); - - columnNames.add(col2.name); + System.out.println(col2); + // remove the trailing ':' from the family name + col2.name.limit(col2.name.limit() - 1); + columnNames.add(col2.name.slice()); } System.out.println("Starting scanner..."); @@ -340,7 +338,7 @@ private void run() throws Exception { List entry = client.scannerGet(scanner); if (entry.isEmpty()) { - System.out.println("Scanner finished"); + System.out.println("Scanner finished..."); break; } diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/thrift2/DemoClient.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/thrift2/DemoClient.java index e17e6a23ffc1..710971f6dadb 100644 --- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/thrift2/DemoClient.java +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/thrift2/DemoClient.java @@ -26,6 +26,7 @@ import javax.security.auth.Subject; import javax.security.auth.login.LoginContext; import javax.security.sasl.Sasl; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.thrift2.generated.TColumnValue; import org.apache.hadoop.hbase.thrift2.generated.TGet; @@ -62,7 +63,7 @@ public static void main(String[] args) throws Exception { if (args.length >= 2) { port = Integer.parseInt(args[1]); } - org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create(); + Configuration conf = HBaseConfiguration.create(); String principal = conf.get("hbase.thrift.kerberos.principal"); if (principal != null) { secure = true; @@ -139,12 +140,12 @@ public void run() throws Exception { TResult result = client.get(table, get); - System.out.print("row = " + ClientUtils.utf8(result.getRow())); + System.out.println("row = " + ClientUtils.utf8(result.getRow())); for (TColumnValue resultColumnValue : result.getColumnValues()) { - System.out.print("family = " + ClientUtils.utf8(resultColumnValue.getFamily())); - System.out.print("qualifier = " + ClientUtils.utf8(resultColumnValue.getFamily())); - System.out.print("value = " + ClientUtils.utf8(resultColumnValue.getValue())); - System.out.print("timestamp = " + resultColumnValue.getTimestamp()); + System.out.println("family = " + ClientUtils.utf8(resultColumnValue.getFamily())); + System.out.println("qualifier = " + ClientUtils.utf8(resultColumnValue.getQualifier())); + System.out.println("value = " + ClientUtils.utf8(resultColumnValue.getValue())); + System.out.println("timestamp = " + resultColumnValue.getTimestamp()); } transport.close(); diff --git a/hbase-examples/src/main/python/requirements.txt b/hbase-examples/src/main/python/requirements.txt new file mode 100644 index 000000000000..5829eb363f18 --- /dev/null +++ b/hbase-examples/src/main/python/requirements.txt @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +six==1.17.0 +thrift==0.14.1 diff --git a/hbase-examples/src/main/python/thrift1/DemoClient.py b/hbase-examples/src/main/python/thrift1/DemoClient.py index 4051528a94de..82441b35d2b2 100644 --- a/hbase-examples/src/main/python/thrift1/DemoClient.py +++ b/hbase-examples/src/main/python/thrift1/DemoClient.py @@ -1,222 +1,245 @@ -#!/usr/bin/python -''' - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -''' +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" import sys import time -import os - -# Modify this import path to point to the correct location to thrift. -thrift_path = os.path.abspath('/Users/sergey/Downloads/thrift/lib/py/build/lib.macosx-10.8-intel-2.7') -sys.path.append(thrift_path) -gen_py_path = os.path.abspath('gen-py') -sys.path.append(gen_py_path) - -from thrift import Thrift from thrift.transport import TSocket, TTransport from thrift.protocol import TBinaryProtocol -from hbase.ttypes import TThriftServerType -from hbase.Hbase import Client, ColumnDescriptor, Mutation +from gen_py.hbase import ttypes +from gen_py.hbase.Hbase import Client, ColumnDescriptor, Mutation + def printVersions(row, versions): - print "row: " + row + ", values: ", - for cell in versions: - print cell.value + "; ", - print + print("row: " + row.decode() + ", values: ", end="") + for cell in versions: + print(cell.value.decode() + "; ", end="") + print() -def printRow(entry): - print "row: " + entry.row + ", cols:", - for k in sorted(entry.columns): - print k + " => " + entry.columns[k].value, - print +def printRow(row_result): + for res in row_result: + sorted_cols = dict(sorted(res.columns.items())) + row_str = "" + for k, v in sorted_cols.items(): + # ignore error to prevent UnicodeDecodeError: 'utf-8' codec can't decode byte 0xfc in + # position 4: invalid start byte + row_str += ( + f"{k.decode('utf-8')} => {v.value.decode('utf-8', errors='ignore')}; " + ) + print(f"row: {res.row.decode('utf-8', errors='ignore')}, cols: {row_str}") -def demo_client(host, port, is_framed_transport): - # Make socket - socket = TSocket.TSocket(host, port) - - # Make transport - if is_framed_transport: - transport = TTransport.TFramedTransport(socket) - else: - transport = TTransport.TBufferedTransport(socket) - - # Wrap in a protocol - protocol = TBinaryProtocol.TBinaryProtocol(transport) - - # Create a client to use the protocol encoder - client = Client(protocol) - - # Connect! - transport.open() - - # Check Thrift Server Type - serverType = client.getThriftServerType() - if serverType != TThriftServerType.ONE: - raise Exception("Mismatch between client and server, server type is %s" % serverType) - - t = "demo_table" - - # - # Scan all tables, look for the demo table and delete it. - # - print "scanning tables..." - for table in client.getTableNames(): - print " found: %s" %(table) - if table == t: - if client.isTableEnabled(table): - print " disabling table: %s" %(t) - client.disableTable(table) - print " deleting table: %s" %(t) - client.deleteTable(table) - - columns = [] - col = ColumnDescriptor() - col.name = 'entry:' - col.maxVersions = 10 - columns.append(col) - col = ColumnDescriptor() - col.name = 'unused:' - columns.append(col) - - try: - print "creating table: %s" %(t) - client.createTable(t, columns) - except AlreadyExists, ae: - print "WARN: " + ae.message - - cols = client.getColumnDescriptors(t) - print "column families in %s" %(t) - for col_name in cols.keys(): - col = cols[col_name] - print " column: %s, maxVer: %d" % (col.name, col.maxVersions) - - dummy_attributes = {} - # - # Test UTF-8 handling - # - invalid = "foo-\xfc\xa1\xa1\xa1\xa1\xa1" - valid = "foo-\xE7\x94\x9F\xE3\x83\x93\xE3\x83\xBC\xE3\x83\xAB"; - - # non-utf8 is fine for data - mutations = [Mutation(column="entry:foo",value=invalid)] - print str(mutations) - client.mutateRow(t, "foo", mutations, dummy_attributes) - - # try empty strings - mutations = [Mutation(column="entry:", value="")] - client.mutateRow(t, "", mutations, dummy_attributes) - - # this row name is valid utf8 - mutations = [Mutation(column="entry:foo", value=valid)] - client.mutateRow(t, valid, mutations, dummy_attributes) - - # non-utf8 is not allowed in row names - try: - mutations = [Mutation(column="entry:foo", value=invalid)] - client.mutateRow(t, invalid, mutations, dummy_attributes) - except ttypes.IOError, e: - print 'expected exception: %s' %(e.message) - - # Run a scanner on the rows we just created - print "Starting scanner..." - scanner = client.scannerOpen(t, "", ["entry:"], dummy_attributes) - - r = client.scannerGet(scanner) - while r: - printRow(r[0]) - r = client.scannerGet(scanner) - print "Scanner finished" - - # - # Run some operations on a bunch of rows. - # - for e in range(100, 0, -1): - # format row keys as "00000" to "00100" - row = "%0.5d" % (e) - - mutations = [Mutation(column="unused:", value="DELETE_ME")] - client.mutateRow(t, row, mutations, dummy_attributes) - printRow(client.getRow(t, row, dummy_attributes)[0]) - client.deleteAllRow(t, row, dummy_attributes) - - mutations = [Mutation(column="entry:num", value="0"), - Mutation(column="entry:foo", value="FOO")] - client.mutateRow(t, row, mutations, dummy_attributes) - printRow(client.getRow(t, row, dummy_attributes)[0]); - - mutations = [Mutation(column="entry:foo",isDelete=True), - Mutation(column="entry:num",value="-1")] - client.mutateRow(t, row, mutations, dummy_attributes) - printRow(client.getRow(t, row, dummy_attributes)[0]) - - mutations = [Mutation(column="entry:num", value=str(e)), - Mutation(column="entry:sqr", value=str(e*e))] - client.mutateRow(t, row, mutations, dummy_attributes) - printRow(client.getRow(t, row, dummy_attributes)[0]) - - time.sleep(0.05) - - mutations = [Mutation(column="entry:num",value="-999"), - Mutation(column="entry:sqr",isDelete=True)] - client.mutateRowTs(t, row, mutations, 1, dummy_attributes) # shouldn't override latest - printRow(client.getRow(t, row, dummy_attributes)[0]) - - versions = client.getVer(t, row, "entry:num", 10, dummy_attributes) - printVersions(row, versions) - if len(versions) != 3: - print("FATAL: wrong # of versions") - sys.exit(-1) - - r = client.get(t, row, "entry:foo", dummy_attributes) - # just to be explicit, we get lists back, if it's empty there was no matching row. - if len(r) > 0: - raise "shouldn't get here!" - - columnNames = [] - for (col, desc) in client.getColumnDescriptors(t).items(): - print "column with name: "+desc.name - print desc - columnNames.append(desc.name+":") - - print "Starting scanner..." - scanner = client.scannerOpenWithStop(t, "00020", "00040", columnNames, dummy_attributes) - - r = client.scannerGet(scanner) - while r: - printRow(r[0]) - r = client.scannerGet(scanner) - - client.scannerClose(scanner) - print "Scanner finished" - - transport.close() - - -if __name__ == '__main__': - - import sys - if len(sys.argv) < 3: - print 'usage: %s ' % __file__ - sys.exit(1) - - host = sys.argv[1] - port = sys.argv[2] - is_framed_transport = False - demo_client(host, port, is_framed_transport) +def demo_client(host, port, is_framed_transport): + # Make socket + socket = TSocket.TSocket(host, port) + + # Make transport + if is_framed_transport: + transport = TTransport.TFramedTransport(socket) + else: + transport = TTransport.TBufferedTransport(socket) + + # Wrap in a protocol + protocol = TBinaryProtocol.TBinaryProtocol(transport) + + # Create a client to use the protocol encoder + client = Client(protocol) + + # Connect! + transport.open() + + # Check Thrift Server Type + serverType = client.getThriftServerType() + if serverType != ttypes.TThriftServerType.ONE: + raise RuntimeError( + f"Mismatch between client and server, server type is {serverType}" + ) + + demo_table = "demo_table".encode() + disable_table = "disabled_table".encode() + + # Scan all tables, look for the demo table and delete it. + print("scanning tables...") + for table in client.getTableNames(): + print(f" found: {table.decode()}") + if table in {demo_table, disable_table}: + if client.isTableEnabled(table): + print(f" disabling table: {table.decode()}") + client.disableTable(table) + print(f" deleting table: {table.decode()}") + client.deleteTable(table) + + # Create the demo table with two column families, entry: and unused: + columns = [] + col = ColumnDescriptor() + col.name = "entry:".encode() + col.maxVersions = 10 + columns.append(col) + col = ColumnDescriptor() + col.name = "unused:".encode() + columns.append(col) + + print(f"creating table: {demo_table.decode()}") + try: + client.createTable(demo_table, columns) + client.createTable(disable_table, columns) + except ttypes.AlreadyExists as ae: + print("WARN: " + ae.message) + + cols = client.getColumnDescriptors(demo_table) + print(f"column families in {demo_table.decode()}: ") + for col_name in cols.keys(): + col = cols[col_name] + print(f" column: {col.name.decode()}, maxVer: {col.maxVersions}") + + if client.isTableEnabled(disable_table): + print(f"disabling table: {disable_table.decode()}") + client.disableTable(disable_table) + + dummy_attributes = {} + + # Test UTF-8 handling + invalid = b"foo-\xfc\xa1\xa1\xa1\xa1" + valid = b"foo-\xe7\x94\x9f\xe3\x83\x93\xe3\x83\xbc\xe3\x83\xab" + + # non-utf8 is fine for data + mutations = [Mutation(column="entry:foo".encode(), value=invalid, writeToWAL=False)] + client.mutateRow(demo_table, "foo".encode(), mutations, dummy_attributes) + + # this row name is valid utf8 + mutations = [Mutation(column="entry:foo".encode(), value=valid, writeToWAL=False)] + client.mutateRow(demo_table, valid, mutations, dummy_attributes) + + # non-utf8 is now allowed in row names because HBase stores values as binary + mutations = [Mutation(column="entry:foo".encode(), value=invalid, writeToWAL=False)] + client.mutateRow(demo_table, invalid, mutations, dummy_attributes) + + # Run a scanner on the rows we just created + print("Starting scanner...") + scanner = client.scannerOpen( + demo_table, "".encode(), ["entry".encode()], dummy_attributes + ) + + while True: + r = client.scannerGet(scanner) + if not r: + break + printRow(r) + print("Scanner finished...") + + # Run some operations on a bunch of rows. + for e in range(100, -1, -1): + # format row keys as "00000" to "00100" + row = f"{e:05}".encode() + + mutations = [ + Mutation( + column="unused:".encode(), value="DELETE_ME".encode(), writeToWAL=False + ) + ] + client.mutateRow(demo_table, row, mutations, dummy_attributes) + printRow(client.getRow(demo_table, row, dummy_attributes)) + client.deleteAllRow(demo_table, row, dummy_attributes) + + # sleep to force later timestamp + time.sleep(0.05) + + mutations = [ + Mutation(column="entry:num".encode(), value="0".encode(), writeToWAL=False), + Mutation( + column="entry:foo".encode(), value="FOO".encode(), writeToWAL=False + ), + ] + client.mutateRow(demo_table, row, mutations, dummy_attributes) + printRow(client.getRow(demo_table, row, dummy_attributes)) + + mutations = [ + Mutation(column="entry:foo".encode(), isDelete=True), + Mutation(column="entry:num".encode(), value="-1".encode()), + ] + client.mutateRow(demo_table, row, mutations, dummy_attributes) + printRow(client.getRow(demo_table, row, dummy_attributes)) + + mutations = [ + Mutation( + column="entry:num".encode(), value=str(e).encode(), writeToWAL=False + ), + Mutation( + column="entry:sqr".encode(), value=str(e * e).encode(), writeToWAL=False + ), + ] + client.mutateRow(demo_table, row, mutations, dummy_attributes) + printRow(client.getRow(demo_table, row, dummy_attributes)) + + # sleep to force later timestamp + time.sleep(0.05) + + mutations = [ + Mutation(column="entry:num".encode(), value="-999".encode()), + Mutation(column="entry:sqr".encode(), isDelete=True), + ] + # shouldn't override latest + client.mutateRowTs(demo_table, row, mutations, 1, dummy_attributes) + printRow(client.getRow(demo_table, row, dummy_attributes)) + + versions = client.getVer( + demo_table, row, "entry:num".encode(), 10, dummy_attributes + ) + printVersions(row, versions) + if len(versions) != 3: + print("FATAL: wrong # of versions") + sys.exit(-1) + + r = client.get(demo_table, row, "entry:foo".encode(), dummy_attributes) + # just to be explicit, we get lists back, if it's empty there was no matching row. + if len(r) > 0: + print("FATAL: shouldn't get here") + sys.exit(-1) + print() + + # scan all rows/columnNames + columnNames = [] + for col, desc in client.getColumnDescriptors(demo_table).items(): + print(f"column with name: {desc.name.decode()}") + print(desc) + columnNames.append(desc.name[:-1]) + + print("\nStarting scanner...") + scanner = client.scannerOpenWithStop( + demo_table, "00020".encode(), "00040".encode(), columnNames, dummy_attributes + ) + while True: + r = client.scannerGet(scanner) + if not r: + break + printRow(r) + print("Scanner finished...") + transport.close() + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print(f"usage: {__file__} ") + sys.exit(1) + + default_host = sys.argv[1] + default_port = sys.argv[2] + + default_is_framed_transport = False + demo_client(default_host, default_port, default_is_framed_transport) diff --git a/hbase-examples/src/main/python/thrift1/demo_hbase_thrift_over_http_tls.py b/hbase-examples/src/main/python/thrift1/demo_hbase_thrift_over_http_tls.py new file mode 100755 index 000000000000..357de8a88ac7 --- /dev/null +++ b/hbase-examples/src/main/python/thrift1/demo_hbase_thrift_over_http_tls.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +''' + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +''' +# +# test thrift server over HTTP with SSL in place +# +# presumes thrift python lib is installed +# presumes you have access to hbase thrift binding (i.e. you add gen_py to PYTHONPATH) +# presumes thrift proxy is running on port 9090 +# presumes thrift proxy is running over https +# presumes access to create and use tables in a namespace 'test' +# +# usage: +# ./demo_hbase_thrift_over_http_tls.py host-running-thrift1.example.com +import sys + +from thrift.transport import THttpClient +from thrift.protocol import TBinaryProtocol +from gen_py.hbase import Hbase +from gen_py.hbase.ttypes import ColumnDescriptor +from gen_py.hbase.ttypes import Mutation + +print("[INFO] setup connection") +transport = THttpClient.THttpClient(f"https://{sys.argv[1]}:9090") +client = Hbase.Client(TBinaryProtocol.TBinaryProtocol(transport)) + +table='test:thrift_proxy_demo' + +print("[INFO] start client") +transport.open() + +print("[INFO] list the current tables") +print(client.getTableNames()) + +print("[INFO] create a table, place some data") +client.createTable(table, [ColumnDescriptor(name ='family1:')]) +client.mutateRow(table, 'row1', + [Mutation(column = 'family1:cq1', value = 'foo'), + Mutation(column = 'family1:cq2', value = 'foo')], {}) +client.mutateRow(table, 'row2', + [Mutation(column = 'family1:cq1', value = 'bar'), + Mutation(column = 'family1:cq2', value = 'bar')], {}) +client.mutateRow(table, 'row3', + [Mutation(column = 'family1:cq1', value = 'foo'), + Mutation(column = 'family1:cq2', value = 'foo')], {}) +client.mutateRow(table, 'row4', + [Mutation(column = 'family1:cq1', value = 'bar'), + Mutation(column = 'family1:cq2', value = 'bar')], {}) + +print("[INFO] scan") +scan_id = client.scannerOpen(table, 'row1', [], {}) +for row in client.scannerGetList(scan_id, 25): + print(row) +client.scannerClose(scan_id) +print("[INFO] get") +for row in client.getRow(table, 'row3', {}): + print(row) + +print("[INFO] clean up") +client.disableTable(table) +client.deleteTable(table) diff --git a/hbase-examples/src/main/python/thrift1/gen-py/hbase/__init__.py b/hbase-examples/src/main/python/thrift1/gen-py/hbase/__init__.py deleted file mode 100644 index 31dc15ce46e1..000000000000 --- a/hbase-examples/src/main/python/thrift1/gen-py/hbase/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['ttypes', 'constants', 'Hbase'] diff --git a/hbase-examples/src/main/python/thrift1/gen-py/hbase/Hbase-remote b/hbase-examples/src/main/python/thrift1/gen_py/hbase/Hbase-remote similarity index 100% rename from hbase-examples/src/main/python/thrift1/gen-py/hbase/Hbase-remote rename to hbase-examples/src/main/python/thrift1/gen_py/hbase/Hbase-remote diff --git a/hbase-examples/src/main/python/thrift1/gen-py/hbase/Hbase.py b/hbase-examples/src/main/python/thrift1/gen_py/hbase/Hbase.py similarity index 100% rename from hbase-examples/src/main/python/thrift1/gen-py/hbase/Hbase.py rename to hbase-examples/src/main/python/thrift1/gen_py/hbase/Hbase.py diff --git a/hbase-examples/src/main/python/thrift1/gen_py/hbase/__init__.py b/hbase-examples/src/main/python/thrift1/gen_py/hbase/__init__.py new file mode 100644 index 000000000000..fdadcfc1e8cb --- /dev/null +++ b/hbase-examples/src/main/python/thrift1/gen_py/hbase/__init__.py @@ -0,0 +1,18 @@ +''' + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +''' +__all__ = ['ttypes', 'constants', 'Hbase'] diff --git a/hbase-examples/src/main/python/thrift1/gen-py/hbase/constants.py b/hbase-examples/src/main/python/thrift1/gen_py/hbase/constants.py similarity index 100% rename from hbase-examples/src/main/python/thrift1/gen-py/hbase/constants.py rename to hbase-examples/src/main/python/thrift1/gen_py/hbase/constants.py diff --git a/hbase-examples/src/main/python/thrift1/gen-py/hbase/ttypes.py b/hbase-examples/src/main/python/thrift1/gen_py/hbase/ttypes.py similarity index 100% rename from hbase-examples/src/main/python/thrift1/gen-py/hbase/ttypes.py rename to hbase-examples/src/main/python/thrift1/gen_py/hbase/ttypes.py diff --git a/hbase-examples/src/main/python/thrift2/DemoClient.py b/hbase-examples/src/main/python/thrift2/DemoClient.py index 4e8107a1b5c3..315edbc7ad7d 100644 --- a/hbase-examples/src/main/python/thrift2/DemoClient.py +++ b/hbase-examples/src/main/python/thrift2/DemoClient.py @@ -1,81 +1,103 @@ +#!/usr/bin/env python """ - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. """ + # Instructions: # 1. Run Thrift to generate the python module hbase # thrift --gen py ../../../../../hbase-thrift/src/main/resources/org/apache/hadoop \ # /hbase/thrift2/hbase.thrift # 2. Create a directory of your choosing that contains: # a. This file (DemoClient.py). -# b. The directory gen-py/hbase (generated by instruction step 1). -# 3. pip install thrift==0.9.3 +# b. The directory gen_py/hbase (generated by instruction step 1). +# 3. pip install thrift==0.14.1 # 4. Create a table call "example", with a family called "family1" using the hbase shell. # 5. Start the hbase thrift2 server # bin/hbase thrift2 start # 6. Execute {python DemoClient.py}. import sys -import os -import time - from thrift.transport import TTransport from thrift.transport import TSocket -from thrift.transport import THttpClient from thrift.protocol import TBinaryProtocol - -# Add path for local "gen-py/hbase" for the pre-generated module -gen_py_path = os.path.abspath('gen-py') -sys.path.append(gen_py_path) -from hbase import THBaseService -from hbase.ttypes import * - -print "Thrift2 Demo" -print "This demo assumes you have a table called \"example\" with a column family called \"family1\"" - -host = "localhost" -port = 9090 -framed = False - -socket = TSocket.TSocket(host, port) -if framed: - transport = TTransport.TFramedTransport(socket) -else: - transport = TTransport.TBufferedTransport(socket) -protocol = TBinaryProtocol.TBinaryProtocol(transport) -client = THBaseService.Client(protocol) - -transport.open() - -# Check Thrift Server Type -serverType = client.getThriftServerType() -if serverType != TThriftServerType.TWO: - raise Exception("Mismatch between client and server, server type is %s" % serverType) - -table = "example" - -put = TPut(row="row1", columnValues=[TColumnValue(family="family1",qualifier="qualifier1",value="value1")]) -print "Putting:", put -client.put(table, put) - -get = TGet(row="row1") -print "Getting:", get -result = client.get(table, get) - -print "Result:", result - -transport.close() +from gen_py.hbase import THBaseService +from gen_py.hbase import ttypes + + +def run(host, port, framed=False): + + socket = TSocket.TSocket(host, port) + if framed: + transport = TTransport.TFramedTransport(socket) + else: + transport = TTransport.TBufferedTransport(socket) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + client = THBaseService.Client(protocol) + + transport.open() + + # Check Thrift Server Type + serverType = client.getThriftServerType() + if serverType != ttypes.TThriftServerType.TWO: + raise RuntimeError( + f"Mismatch between client and server, server type is {serverType}" + ) + + table = "example".encode() + + t_column_values = ttypes.TColumnValue( + family="family1".encode(), + qualifier="qualifier1".encode(), + value="value1".encode(), + ) + put = ttypes.TPut(row="row1".encode(), columnValues=[t_column_values]) + print("\nPutting:", put) + client.put(table, put) + + get = ttypes.TGet(row="row1".encode()) + print("\nGetting:", get) + result = client.get(table, get) + + print("\nResult:", result) + print(f"row = {result.row.decode()}") + for column_value in result.columnValues: + print(f"family = {column_value.family.decode()}") + print(f"qualifier = {column_value.qualifier.decode()}") + print(f"value = {column_value.value.decode()}") + print(f"timestamp = {column_value.timestamp}") + + transport.close() + + +if __name__ == "__main__": + print("Thrift2 Demo") + print("Usage: DemoClient [host=localhost] [port=9090]") + print( + "This demo assumes you have a table called " + '"example" with a column family called "family1"' + ) + + default_host = "localhost" + default_port = 9090 + is_framed_transport = False + + if len(sys.argv) >= 2: + default_host = sys.argv[1] + if len(sys.argv) >= 3: + default_port = int(sys.argv[2]) + + run(default_host, default_port, is_framed_transport) diff --git a/hbase-examples/src/main/python/thrift2/gen-py/hbase/__init__.py b/hbase-examples/src/main/python/thrift2/gen-py/hbase/__init__.py deleted file mode 100644 index 378715da5e5e..000000000000 --- a/hbase-examples/src/main/python/thrift2/gen-py/hbase/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['ttypes', 'constants', 'THBaseService'] diff --git a/hbase-examples/src/main/python/thrift2/gen-py/hbase/THBaseService-remote b/hbase-examples/src/main/python/thrift2/gen_py/hbase/THBaseService-remote similarity index 100% rename from hbase-examples/src/main/python/thrift2/gen-py/hbase/THBaseService-remote rename to hbase-examples/src/main/python/thrift2/gen_py/hbase/THBaseService-remote diff --git a/hbase-examples/src/main/python/thrift2/gen-py/hbase/THBaseService.py b/hbase-examples/src/main/python/thrift2/gen_py/hbase/THBaseService.py similarity index 100% rename from hbase-examples/src/main/python/thrift2/gen-py/hbase/THBaseService.py rename to hbase-examples/src/main/python/thrift2/gen_py/hbase/THBaseService.py diff --git a/hbase-examples/src/main/python/thrift2/gen_py/hbase/__init__.py b/hbase-examples/src/main/python/thrift2/gen_py/hbase/__init__.py new file mode 100644 index 000000000000..846ef4003735 --- /dev/null +++ b/hbase-examples/src/main/python/thrift2/gen_py/hbase/__init__.py @@ -0,0 +1,18 @@ +''' + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +''' +__all__ = ['ttypes', 'constants', 'THBaseService'] diff --git a/hbase-examples/src/main/python/thrift2/gen-py/hbase/constants.py b/hbase-examples/src/main/python/thrift2/gen_py/hbase/constants.py similarity index 100% rename from hbase-examples/src/main/python/thrift2/gen-py/hbase/constants.py rename to hbase-examples/src/main/python/thrift2/gen_py/hbase/constants.py diff --git a/hbase-examples/src/main/python/thrift2/gen-py/hbase/ttypes.py b/hbase-examples/src/main/python/thrift2/gen_py/hbase/ttypes.py similarity index 100% rename from hbase-examples/src/main/python/thrift2/gen-py/hbase/ttypes.py rename to hbase-examples/src/main/python/thrift2/gen_py/hbase/ttypes.py diff --git a/hbase-examples/src/main/resources/hbase-site.xml b/hbase-examples/src/main/resources/hbase-site.xml new file mode 100644 index 000000000000..ab4d1cd0b733 --- /dev/null +++ b/hbase-examples/src/main/resources/hbase-site.xml @@ -0,0 +1,28 @@ + + + + + + hbase.defaults.for.version.skip + true + + diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestAsyncClientExample.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestAsyncClientExample.java index 5ce943e01b1e..b904ecf0a99e 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestAsyncClientExample.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestAsyncClientExample.java @@ -17,39 +17,34 @@ */ package org.apache.hadoop.hbase.client.example; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.ToolRunner; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestAsyncClientExample { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAsyncClientExample.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final TableName TABLE_NAME = TableName.valueOf("test"); - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.startMiniCluster(1); UTIL.createTable(TABLE_NAME, Bytes.toBytes("d")); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { UTIL.shutdownMiniCluster(); } diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestHttpProxyExample.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestHttpProxyExample.java index b3f02073400d..96374cb4f1b1 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestHttpProxyExample.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestHttpProxyExample.java @@ -17,10 +17,9 @@ */ package org.apache.hadoop.hbase.client.example; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.charset.StandardCharsets; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.ClientTests; @@ -34,21 +33,17 @@ import org.apache.http.entity.ContentType; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestHttpProxyExample { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHttpProxyExample.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final TableName TABLE_NAME = TableName.valueOf("test"); @@ -67,7 +62,7 @@ public class TestHttpProxyExample { private static int PORT; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.startMiniCluster(1); UTIL.createTable(TABLE_NAME, Bytes.toBytes(FAMILY)); @@ -76,7 +71,7 @@ public static void setUp() throws Exception { PORT = PROXY.port(); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { if (PROXY != null) { PROXY.stop(); diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestMultiThreadedClientExample.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestMultiThreadedClientExample.java index 0db1b229361e..4bc7572ed568 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestMultiThreadedClientExample.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestMultiThreadedClientExample.java @@ -17,23 +17,22 @@ */ package org.apache.hadoop.hbase.client.example; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestMultiThreadedClientExample { private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -43,17 +42,13 @@ public class TestMultiThreadedClientExample { private static byte[] familyName = Bytes.toBytes("d"); private static byte[] columnName = Bytes.toBytes("col"); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMultiThreadedClientExample.class); - - @BeforeClass + @BeforeAll public static void setup() throws Exception { TEST_UTIL.startMiniCluster(1); table = TEST_UTIL.createTable(MY_TABLE_NAME, familyName); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { TEST_UTIL.deleteTable(MY_TABLE_NAME); TEST_UTIL.shutdownMiniCluster(); diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java index b5eac9bacbf9..5ef27aae9e3c 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java @@ -17,26 +17,22 @@ */ package org.apache.hadoop.hbase.client.example; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.coprocessor.example.TestRefreshHFilesBase; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ClientTests.class, MediumTests.class }) +@Tag(ClientTests.TAG) +@Tag(MediumTests.TAG) public class TestRefreshHFilesClient extends TestRefreshHFilesBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRefreshHFilesClient.class); - @BeforeClass + @BeforeAll public static void setUp() { setUp(HRegion.class.getName()); } diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java index 4b44ed245733..dff65a003c2f 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.HFileTestUtil; -import org.junit.After; +import org.junit.jupiter.api.AfterEach; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +70,7 @@ public static void setUp(String regionImpl) { } } - @After + @AfterEach public void tearDown() throws Exception { HTU.shutdownMiniCluster(); } diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java index 4d6a6d3b1aba..4ed9f924a43c 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hbase.coprocessor.example; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.ArrayList; @@ -26,7 +27,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; @@ -38,15 +38,11 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.wal.WAL; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(MediumTests.class) +@Tag(MediumTests.TAG) public class TestRefreshHFilesEndpoint extends TestRefreshHFilesBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRefreshHFilesEndpoint.class); @Test public void testRefreshRegionHFilesEndpoint() throws Exception { @@ -57,10 +53,10 @@ public void testRefreshRegionHFilesEndpoint() throws Exception { assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY)); } - @Test(expected = IOException.class) + @Test public void testRefreshRegionHFilesEndpointWithException() throws IOException { setUp(HRegionForRefreshHFilesEP.class.getName()); - callRefreshRegionHFilesEndPoint(); + assertThrows(IOException.class, () -> callRefreshRegionHFilesEndPoint()); } private void callRefreshRegionHFilesEndPoint() throws IOException { diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestScanModifyingObserver.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestScanModifyingObserver.java index 05e8a1cb74b2..41d5d2d49f9a 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestScanModifyingObserver.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestScanModifyingObserver.java @@ -17,15 +17,14 @@ */ package org.apache.hadoop.hbase.coprocessor.example; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -39,19 +38,15 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestScanModifyingObserver { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestScanModifyingObserver.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final TableName NAME = TableName.valueOf("TestScanModifications"); private static final byte[] FAMILY = Bytes.toBytes("f"); @@ -63,7 +58,7 @@ public class TestScanModifyingObserver { private static final byte[] EXPLICIT_VAL = Bytes.toBytes("provided"); private static final byte[] IMPLICIT_VAL = Bytes.toBytes("implicit"); - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.startMiniCluster(1); UTIL.getAdmin() @@ -74,7 +69,7 @@ public static void setUp() throws Exception { .setColumnFamily(CFD).build()); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { UTIL.shutdownMiniCluster(); } @@ -101,7 +96,7 @@ public void test() throws IOException { try (ResultScanner scanner = t.getScanner(s)) { for (int i = 0; i < NUM_ROWS; i++) { Result result = scanner.next(); - assertNotNull("The " + (i + 1) + "th result was unexpectedly null", result); + assertNotNull(result, "The " + (i + 1) + "th result was unexpectedly null"); assertEquals(2, result.getFamilyMap(FAMILY).size()); assertArrayEquals(Bytes.toBytes(i + 1), result.getRow()); assertArrayEquals(EXPLICIT_VAL, result.getValue(FAMILY, EXPLICIT_QUAL)); diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestValueReplacingCompaction.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestValueReplacingCompaction.java index 1e5c0b998321..30b3997c1f2d 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestValueReplacingCompaction.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestValueReplacingCompaction.java @@ -17,15 +17,14 @@ */ package org.apache.hadoop.hbase.coprocessor.example; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -39,19 +38,15 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestValueReplacingCompaction { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestValueReplacingCompaction.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final TableName NAME = TableName.valueOf("TestValueReplacement"); private static final byte[] FAMILY = Bytes.toBytes("f"); @@ -62,7 +57,7 @@ public class TestValueReplacingCompaction { private static final String value = "foo"; private static final String replacedValue = "bar"; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.startMiniCluster(1); UTIL.getAdmin() @@ -73,7 +68,7 @@ public static void setUp() throws Exception { .build()); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { UTIL.shutdownMiniCluster(); } @@ -104,7 +99,7 @@ public void test() throws IOException, InterruptedException { try (ResultScanner scanner = t.getScanner(s)) { for (int i = 0; i < NUM_ROWS; i++) { Result result = scanner.next(); - assertNotNull("The " + (i + 1) + "th result was unexpectedly null", result); + assertNotNull(result, "The " + (i + 1) + "th result was unexpectedly null"); assertEquals(1, result.getFamilyMap(FAMILY).size()); assertArrayEquals(Bytes.toBytes(i + 1), result.getRow()); assertArrayEquals(Bytes.toBytes(replacedValue), result.getValue(FAMILY, QUALIFIER)); diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestWriteHeavyIncrementObserver.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestWriteHeavyIncrementObserver.java index f0d7591dadd9..270f15f2d1f5 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestWriteHeavyIncrementObserver.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestWriteHeavyIncrementObserver.java @@ -17,10 +17,9 @@ */ package org.apache.hadoop.hbase.coprocessor.example; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -30,19 +29,16 @@ import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestWriteHeavyIncrementObserver extends WriteHeavyIncrementObserverTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestWriteHeavyIncrementObserver.class); - - @BeforeClass + @BeforeAll public static void setUp() throws Exception { WriteHeavyIncrementObserverTestBase.setUp(); UTIL.getAdmin() @@ -52,6 +48,14 @@ public static void setUp() throws Exception { TABLE = UTIL.getConnection().getTable(NAME); } + @AfterAll + public static void tearDown() throws Exception { + if (TABLE != null) { + TABLE.close(); + } + UTIL.shutdownMiniCluster(); + } + @Test public void test() throws Exception { doIncrement(0); diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestWriteHeavyIncrementObserverWithMemStoreCompaction.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestWriteHeavyIncrementObserverWithMemStoreCompaction.java index 22409bebd0ec..0314a1719681 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestWriteHeavyIncrementObserverWithMemStoreCompaction.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestWriteHeavyIncrementObserverWithMemStoreCompaction.java @@ -17,9 +17,8 @@ */ package org.apache.hadoop.hbase.coprocessor.example; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.MemoryCompactionPolicy; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; @@ -27,20 +26,17 @@ import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestWriteHeavyIncrementObserverWithMemStoreCompaction extends WriteHeavyIncrementObserverTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestWriteHeavyIncrementObserverWithMemStoreCompaction.class); - - @BeforeClass + @BeforeAll public static void setUp() throws Exception { WriteHeavyIncrementObserverTestBase.setUp(); UTIL.getAdmin() @@ -52,6 +48,14 @@ public static void setUp() throws Exception { TABLE = UTIL.getConnection().getTable(NAME); } + @AfterAll + public static void tearDown() throws Exception { + if (TABLE != null) { + TABLE.close(); + } + UTIL.shutdownMiniCluster(); + } + @Test public void test() throws Exception { // sleep every 10 loops to give memstore compaction enough time to finish before reaching the diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java index 569c429b5ac0..7b8ce36dad2c 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java @@ -17,11 +17,10 @@ */ package org.apache.hadoop.hbase.coprocessor.example; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; @@ -38,19 +37,15 @@ import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ CoprocessorTests.class, MediumTests.class }) +@Tag(CoprocessorTests.TAG) +@Tag(MediumTests.TAG) public class TestZooKeeperScanPolicyObserver { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestZooKeeperScanPolicyObserver.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static TableName NAME = TableName.valueOf("TestCP"); @@ -61,7 +56,7 @@ public class TestZooKeeperScanPolicyObserver { private static Table TABLE; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.startMiniCluster(3); UTIL.getAdmin() @@ -74,7 +69,7 @@ public static void setUp() throws Exception { TABLE = UTIL.getConnection().getTable(NAME); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { if (TABLE != null) { TABLE.close(); diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/WriteHeavyIncrementObserverTestBase.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/WriteHeavyIncrementObserverTestBase.java index b1b00ed3b9d0..265b5db51327 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/WriteHeavyIncrementObserverTestBase.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/WriteHeavyIncrementObserverTestBase.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.coprocessor.example; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.io.InterruptedIOException; @@ -32,8 +32,6 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; public class WriteHeavyIncrementObserverTestBase { @@ -55,7 +53,6 @@ public class WriteHeavyIncrementObserverTestBase { protected static int THREADS = 10; - @BeforeClass public static void setUp() throws Exception { UTIL.getConfiguration().setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 64 * 1024L); UTIL.getConfiguration().setLong("hbase.hregion.memstore.flush.size.limit", 1024L); @@ -64,14 +61,6 @@ public static void setUp() throws Exception { UTIL.startMiniCluster(3); } - @AfterClass - public static void tearDown() throws Exception { - if (TABLE != null) { - TABLE.close(); - } - UTIL.shutdownMiniCluster(); - } - private static void increment(int sleepSteps) throws IOException { for (long i = 1; i <= UPPER; i++) { TABLE.increment(new Increment(ROW).addColumn(FAMILY, CQ1, i).addColumn(FAMILY, CQ2, 2 * i)); diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMapReduceExamples.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMapReduceExamples.java index 382f3672991b..341e2a960f03 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMapReduceExamples.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMapReduceExamples.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.mapreduce; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -29,7 +29,6 @@ import java.io.PrintStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -46,17 +45,14 @@ import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -@Category({ MapReduceTests.class, LargeTests.class }) +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) public class TestMapReduceExamples { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMapReduceExamples.class); private static HBaseTestingUtility util = new HBaseTestingUtility(); diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/security/provider/example/TestShadeSaslAuthenticationProvider.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/security/provider/example/TestShadeSaslAuthenticationProvider.java index 1d0bb40b4406..a68d9837fde3 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/security/provider/example/TestShadeSaslAuthenticationProvider.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/security/provider/example/TestShadeSaslAuthenticationProvider.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.hbase.security.provider.example; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.BufferedWriter; import java.io.File; @@ -40,7 +40,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.LocalHBaseCluster; @@ -67,26 +66,22 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MediumTests.class, SecurityTests.class }) +@Tag(MediumTests.TAG) +@Tag(SecurityTests.TAG) public class TestShadeSaslAuthenticationProvider { + private static final Logger LOG = LoggerFactory.getLogger(TestShadeSaslAuthenticationProvider.class); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestShadeSaslAuthenticationProvider.class); - private static final char[] USER1_PASSWORD = "foobarbaz".toCharArray(); static LocalHBaseCluster createCluster(HBaseTestingUtility util, File keytabFile, MiniKdc kdc, @@ -138,7 +133,7 @@ static void createUserDBFile(FileSystem fs, Path p, Map userData private static File KEYTAB_FILE; private static Path USER_DATABASE_FILE; - @BeforeClass + @BeforeAll public static void setupCluster() throws Exception { KEYTAB_FILE = new File(UTIL.getDataTestDir("keytab").toUri().getPath()); final MiniKdc kdc = UTIL.setupMiniKdc(KEYTAB_FILE); @@ -156,7 +151,7 @@ public static void setupCluster() throws Exception { CLUSTER.startup(); } - @AfterClass + @AfterAll public static void teardownCluster() throws Exception { if (CLUSTER != null) { CLUSTER.shutdown(); @@ -165,14 +160,12 @@ public static void teardownCluster() throws Exception { UTIL.shutdownMiniZKCluster(); } - @Rule - public TestName name = new TestName(); TableName tableName; String clusterId; - @Before - public void createTable() throws Exception { - tableName = TableName.valueOf(name.getMethodName()); + @BeforeEach + public void createTable(TestInfo testInfo) throws Exception { + tableName = TableName.valueOf(testInfo.getTestMethod().get().getName()); // Create a table and write a record as the service user (hbase) UserGroupInformation serviceUgi = UserGroupInformation @@ -215,9 +208,9 @@ public Void run() throws Exception { try (Table t = conn.getTable(tableName)) { Result r = t.get(new Get(Bytes.toBytes("r1"))); assertNotNull(r); - assertFalse("Should have read a non-empty Result", r.isEmpty()); + assertFalse(r.isEmpty(), "Should have read a non-empty Result"); final Cell cell = r.getColumnLatestCell(Bytes.toBytes("f1"), Bytes.toBytes("q1")); - assertTrue("Unexpected value", CellUtil.matchingValue(cell, Bytes.toBytes("1"))); + assertTrue(CellUtil.matchingValue(cell, Bytes.toBytes("1")), "Unexpected value"); return null; } @@ -257,7 +250,7 @@ public Void run() throws Exception { fail("Should not successfully authenticate with HBase"); } catch (Exception e) { LOG.info("Caught exception in negative Master connectivity test", e); - assertEquals("Found unexpected exception", pair.getSecond(), e.getClass()); + assertEquals(pair.getSecond(), e.getClass(), "Found unexpected exception"); } return null; } @@ -273,7 +266,7 @@ public Void run() throws Exception { fail("Should not successfully authenticate with HBase"); } catch (Exception e) { LOG.info("Caught exception in negative RegionServer connectivity test", e); - assertEquals("Found unexpected exception", pair.getSecond(), e.getClass()); + assertEquals(pair.getSecond(), e.getClass(), "Found unexpected exception"); } return null; } diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/types/TestPBCell.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/types/TestPBCell.java index abc2ccd692b8..b3e4fd6c418f 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/types/TestPBCell.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/types/TestPBCell.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hbase.types; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.CellProtos; @@ -33,16 +32,13 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.PositionedByteRange; import org.apache.hadoop.hbase.util.SimplePositionedByteRange; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ SmallTests.class, MiscTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestPBCell { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestPBCell.class); - private static final PBCell CODEC = new PBCell(); /** diff --git a/hbase-extensions/hbase-openssl/pom.xml b/hbase-extensions/hbase-openssl/pom.xml index bbe161921cc6..b1183a1bec76 100644 --- a/hbase-extensions/hbase-openssl/pom.xml +++ b/hbase-extensions/hbase-openssl/pom.xml @@ -49,6 +49,13 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + true + + diff --git a/hbase-external-blockcache/pom.xml b/hbase-external-blockcache/pom.xml index b1afadc9b409..c764514801bb 100644 --- a/hbase-external-blockcache/pom.xml +++ b/hbase-external-blockcache/pom.xml @@ -102,12 +102,21 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test - diff --git a/hbase-external-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestMemcachedBlockCache.java b/hbase-external-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestMemcachedBlockCache.java index 60370f5208b1..30d369ed2b66 100644 --- a/hbase-external-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestMemcachedBlockCache.java +++ b/hbase-external-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestMemcachedBlockCache.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.io.hfile; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; @@ -44,29 +44,24 @@ import net.spy.memcached.ops.OperationStatus; import net.spy.memcached.transcoders.Transcoder; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.io.hfile.CacheTestUtils.HFileBlockPair; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ IOTests.class, SmallTests.class }) +@Tag(IOTests.TAG) +@Tag(SmallTests.TAG) public class TestMemcachedBlockCache { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMemcachedBlockCache.class); - private MemcachedBlockCache cache; private ConcurrentMap backingMap; - @Before + @BeforeEach public void setup() throws Exception { int port = ThreadLocalRandom.current().nextInt(1024, 65536); Configuration conf = new Configuration(); @@ -153,5 +148,4 @@ public void testEviction() throws Exception { } Waiter.waitFor(new Configuration(), 10000, () -> backingMap.size() == 0); } - } diff --git a/hbase-hadoop-compat/pom.xml b/hbase-hadoop-compat/pom.xml index 059ad7b1c1fb..413107c77491 100644 --- a/hbase-hadoop-compat/pom.xml +++ b/hbase-hadoop-compat/pom.xml @@ -65,8 +65,18 @@ hbase-metrics-api - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index c68809a1fddb..0f1619084c1f 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -401,12 +401,22 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String L1_CACHE_HIT_COUNT = "l1CacheHitCount"; String L1_CACHE_HIT_COUNT_DESC = "L1 cache hit count."; + String L1_CACHE_HIT_CAGHING_COUNT = "l1CacheHitCachingCount"; + String L1_CACHE_HIT_CAGHING_COUNT_DESC = + "Count of the hit on the L1 cache, for cacheable requests"; String L1_CACHE_MISS_COUNT = "l1CacheMissCount"; String L1_CACHE_MISS_COUNT_DESC = "L1 cache miss count."; + String L1_CACHE_MISS_CACHING_COUNT = "l1CacheMissCachingCount"; + String L1_CACHE_MISS_CACHING_COUNT_DESC = + "Count of the misses on the L1 cache, for cacheable requests"; String L1_CACHE_HIT_RATIO = "l1CacheHitRatio"; String L1_CACHE_HIT_RATIO_DESC = "L1 cache hit ratio."; + String L1_CACHE_HIT_CACHING_RATIO = "l1CacheHitCachingRatio"; + String L1_CACHE_HIT_CACHING_RATIO_DESC = "Hit Caching ratio of L1 cache"; String L1_CACHE_MISS_RATIO = "l1CacheMissRatio"; String L1_CACHE_MISS_RATIO_DESC = "L1 cache miss ratio."; + String L1_CACHE_MISS_CACHING_RATIO = "l1CacheMissCachingRatio"; + String L1_CACHE_MISS_CACHING_RATIO_DESC = "Miss Caching ratio of L1 cache"; String L2_CACHE_FREE_SIZE = "l2CacheFreeSize"; String L2_CACHE_FREE_SIZE_DESC = "Amount of free bytes in the L2 cache"; String L2_CACHE_SIZE = "l2CacheSize"; @@ -417,12 +427,22 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String L2_CACHE_EVICTION_COUNT_DESC = "Count of blocks evicted from the L2 cache"; String L2_CACHE_HIT_COUNT = "l2CacheHitCount"; String L2_CACHE_HIT_COUNT_DESC = "L2 cache hit count."; + String L2_CACHE_HIT_CACHING_COUNT = "l2CacheHitCachingCount"; + String L2_CACHE_HIT_CACHING_COUNT_DESC = + "Count of the hit on the L2 cache, for cacheable requests"; String L2_CACHE_MISS_COUNT = "l2CacheMissCount"; String L2_CACHE_MISS_COUNT_DESC = "L2 cache miss count."; + String L2_CACHE_MISS_CACHING_COUNT = "l2CacheMissCachingCount"; + String L2_CACHE_MISS_CACHING_COUNT_DESC = + "Count of the misses on the L2 cache, for cacheable requests"; String L2_CACHE_HIT_RATIO = "l2CacheHitRatio"; String L2_CACHE_HIT_RATIO_DESC = "L2 cache hit ratio."; + String L2_CACHE_HIT_CACHING_RATIO = "l2CacheHitCachingRatio"; + String L2_CACHE_HIT_CACHING_RATIO_DESC = "Hit Caching ratio of L2 cache"; String L2_CACHE_MISS_RATIO = "l2CacheMissRatio"; String L2_CACHE_MISS_RATIO_DESC = "L2 cache miss ratio."; + String L2_CACHE_MISS_CACHING_RATIO = "l2CacheMissCachingRatio"; + String L2_CACHE_MISS_CACHING_RATIO_DESC = "Miss Caching ratio of L2 cache"; String RS_START_TIME_NAME = "regionServerStartTime"; String ZOOKEEPER_QUORUM_NAME = "zookeeperQuorum"; String SERVER_NAME_NAME = "serverName"; @@ -533,6 +553,14 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String ZEROCOPY_BYTES_READ = "zeroCopyBytesRead"; String ZEROCOPY_BYTES_READ_DESC = "The number of bytes read through HDFS zero copy"; + String LOCAL_RACK_BYTES_READ = "localRackBytesRead"; + String LOCAL_RACK_BYTES_READ_DESC = + "The number of bytes read from the same rack of the RegionServer, but not the local HDFS DataNode"; + + String REMOTE_RACK_BYTES_READ = "remoteRackBytesRead"; + String REMOTE_RACK_BYTES_READ_DESC = + "The number of bytes read from a different rack from that of the RegionServer"; + String BLOCKED_REQUESTS_COUNT = "blockedRequestCount"; String BLOCKED_REQUESTS_COUNT_DESC = "The number of blocked requests because of memstore size is " + "larger than blockingMemStoreSize"; @@ -640,4 +668,10 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo "Count of scanners which were expired due to scanner lease timeout"; String CURRENT_REGION_CACHE_RATIO = "currentRegionCacheRatio"; String CURRENT_REGION_CACHE_RATIO_DESC = "The percentage of caching completed for this region."; + + String EXCLUDE_DATA_NODES_COUNT = "excludedDataNodesCount"; + String EXCLUDE_DATA_NODES_COUNT_DESC = + "Count of slow/connect error DataNodes excluded during WAL write operation"; + String EXCLUDE_DATA_NODES_DETAILS = "excludedDataNodesDetails"; + String EXCLUDE_DATA_NODES_DETAILS_DESC = "Excluded DataNodes info"; } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 10e71d091f59..f40748206e1e 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -364,21 +364,41 @@ public interface MetricsRegionServerWrapper { */ long getL1CacheHitCount(); + /** + * Hit Caching count of L1 cache, for cacheable requests only. + */ + long getL1CacheHitCachingCount(); + /** * Miss count of L1 cache. */ long getL1CacheMissCount(); + /** + * Miss Caching count of L1 cache, for cacheable requests only. + */ + long getL1CacheMissCachingCount(); + /** * Hit ratio of L1 cache. */ double getL1CacheHitRatio(); + /** + * Hit Caching ratio of L1 cache. + */ + double getL1CacheHitCachingRatio(); + /** * Miss ratio of L1 cache. */ double getL1CacheMissRatio(); + /** + * Miss Caching ratio of L1 cache. + */ + double getL1CacheMissCachingRatio(); + /** * Cache size (bytes) of L2 cache */ @@ -404,21 +424,41 @@ public interface MetricsRegionServerWrapper { */ long getL2CacheHitCount(); + /** + * Hit Caching count of L2 cache, for cacheable requests only. + */ + long getL2CacheHitCachingCount(); + /** * Miss count of L2 cache. */ long getL2CacheMissCount(); + /** + * Miss Caching count of L2 cache, for cacheable requests only. + */ + long getL2CacheMissCachingCount(); + /** * Hit ratio of L2 cache. */ double getL2CacheHitRatio(); + /** + * Hit Caching ratio of L2 cache. + */ + double getL2CacheHitCachingRatio(); + /** * Miss ratio of L2 cache. */ double getL2CacheMissRatio(); + /** + * Miss Caching ratio of L2 cache. + */ + double getL2CacheMissCachingRatio(); + /** * Force a re-computation of the metrics. */ @@ -544,6 +584,10 @@ public interface MetricsRegionServerWrapper { /** Returns Number of bytes read from the local HDFS DataNode. */ long getLocalBytesRead(); + long getLocalRackBytesRead(); + + long getRemoteRackBytesRead(); + /** Returns Number of bytes read locally through HDFS short circuit. */ long getShortCircuitBytesRead(); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java index c3d955592d6a..41267de1981a 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java @@ -56,6 +56,8 @@ public interface MetricsRegionSource extends Comparable { String ROW_READS_ONLY_ON_MEMSTORE_DESC = "Row reads happening completely out of memstore"; String MIXED_ROW_READS = "mixedRowReadsCount"; String MIXED_ROW_READS_ON_STORE_DESC = "Row reads happening out of files and memstore on store"; + String TABLE_DESCRIPTOR_HASH = "tableDescriptorHash"; + String TABLE_DESCRIPTOR_HASH_DESC = "The hash of the current table descriptor"; /** * Close the region's metrics as this region is closing. diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index 4d8a028d89b1..380643d27c29 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -60,6 +60,16 @@ public interface MetricsRegionWrapper { */ long getMemStoreSize(); + /** + * Get the size of the on heap memstore of this region + */ + long getMemStoreHeapSize(); + + /** + * Get the size of the off heap memstore of this region + */ + long getMemStoreOffHeapSize(); + /** * Get the total size of the store files this region server is serving from. */ @@ -161,4 +171,12 @@ public interface MetricsRegionWrapper { /** Returns the number of row reads on memstore and file per store */ Map getMixedRowReadsCount(); + /** + * Returns a hash of the table descriptor that this region was opened with. This hash uniquely + * identifies the table configuration (column families, compression, TTL, block size, etc.) and + * can be used to determine if a region needs to be reopened to pick up descriptor changes. + * @return hex-encoded hash of the serialized TableDescriptor + */ + String getTableDescriptorHash(); + } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java index b64c01aa799e..287a980acc04 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java @@ -53,6 +53,16 @@ public interface MetricsTableWrapperAggregate { */ long getMemStoreSize(String table); + /** + * Get the size of the on heap memstore of this table + */ + long getMemStoreHeapSize(String table); + + /** + * Get the size of the off heap memstore of this table + */ + long getMemStoreOffHeapSize(String table); + /** * Get the store file size against this table */ diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/TestCompatibilitySingletonFactory.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/TestCompatibilitySingletonFactory.java index 10f112407433..7da2c4c6fc66 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/TestCompatibilitySingletonFactory.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/TestCompatibilitySingletonFactory.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import java.util.ArrayList; import java.util.List; @@ -28,17 +28,13 @@ import java.util.concurrent.Future; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestCompatibilitySingletonFactory { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCompatibilitySingletonFactory.class); - private static final int ITERATIONS = 100000; private static class TestCompatibilitySingletonFactoryCallable implements Callable { diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceFactory.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceFactory.java index 6071a7e82001..c51f82053cde 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceFactory.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceFactory.java @@ -17,28 +17,25 @@ */ package org.apache.hadoop.hbase.master; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for the CompatibilitySingletonFactory and building MetricsMasterSource */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsMasterSourceFactory { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsMasterSourceFactory.class); - - @Test(expected = RuntimeException.class) - public void testGetInstanceNoHadoopCompat() throws Exception { + @Test + public void testGetInstance() throws Exception { // This should throw an exception because there is no compat lib on the class path. - CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class); - + assertThrows(RuntimeException.class, + () -> CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class)); } } diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerSourceFactory.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerSourceFactory.java index 598995d846fc..871fb86a9f07 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerSourceFactory.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerSourceFactory.java @@ -17,28 +17,25 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for the CompatibilitySingletonFactory and building MetricsRegionServerSource */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsRegionServerSourceFactory { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsRegionServerSourceFactory.class); - - @Test(expected = RuntimeException.class) - public void testGetInstanceNoHadoopCompat() throws Exception { + @Test + public void testGetInstance() throws Exception { // This should throw an exception because there is no compat lib on the class path. - CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceFactory.class); - + assertThrows(RuntimeException.class, + () -> CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceFactory.class)); } } diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWALSource.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWALSource.java index 10de587db0a1..53deb07eec1c 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWALSource.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWALSource.java @@ -17,25 +17,22 @@ */ package org.apache.hadoop.hbase.regionserver.wal; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsWALSource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsWALSource.class); - - @Test(expected = RuntimeException.class) - public void testGetInstanceNoHadoopCompat() throws Exception { + @Test + public void testGetInstance() throws Exception { // This should throw an exception because there is no compat lib on the class path. - CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); - + assertThrows(RuntimeException.class, + () -> CompatibilitySingletonFactory.getInstance(MetricsWALSource.class)); } } diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactory.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactory.java index 2bdd8e754e5b..7f9d2c593be8 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactory.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactory.java @@ -17,27 +17,25 @@ */ package org.apache.hadoop.hbase.replication.regionserver; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for the CompatibilitySingletonFactory and building MetricsReplicationSource */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsReplicationSourceFactory { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsReplicationSourceFactory.class); - - @Test(expected = RuntimeException.class) - public void testGetInstanceNoHadoopCompat() throws Exception { + @Test + public void testGetInstance() throws Exception { // This should throw an exception because there is no compat lib on the class path. - CompatibilitySingletonFactory.getInstance(MetricsReplicationSource.class); + assertThrows(RuntimeException.class, + () -> CompatibilitySingletonFactory.getInstance(MetricsReplicationSource.class)); } } diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/rest/TestMetricsRESTSource.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/rest/TestMetricsRESTSource.java index e81aaec77d0b..33e1208541ab 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/rest/TestMetricsRESTSource.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/rest/TestMetricsRESTSource.java @@ -17,28 +17,25 @@ */ package org.apache.hadoop.hbase.rest; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test of Rest Metrics Source interface. */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsRESTSource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsRESTSource.class); - - @Test(expected = RuntimeException.class) + @Test public void testGetInstanceNoHadoopCompat() throws Exception { // This should throw an exception because there is no compat lib on the class path. - CompatibilitySingletonFactory.getInstance(MetricsRESTSource.class); + assertThrows(RuntimeException.class, + () -> CompatibilitySingletonFactory.getInstance(MetricsRESTSource.class)); } - } diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/thrift/TestMetricsThriftServerSourceFactory.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/thrift/TestMetricsThriftServerSourceFactory.java index 23a4ef8e7424..70ec2bf838e2 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/thrift/TestMetricsThriftServerSourceFactory.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/thrift/TestMetricsThriftServerSourceFactory.java @@ -17,28 +17,25 @@ */ package org.apache.hadoop.hbase.thrift; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for the interface of MetricsThriftServerSourceFactory */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsThriftServerSourceFactory { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsThriftServerSourceFactory.class); - - @Test(expected = RuntimeException.class) - public void testGetInstanceNoHadoopCompat() throws RuntimeException { + @Test + public void testGetInstance() { // This should throw an exception because there is no compat lib on the class path. - CompatibilitySingletonFactory.getInstance(MetricsThriftServerSourceFactory.class); + assertThrows(RuntimeException.class, + () -> CompatibilitySingletonFactory.getInstance(MetricsThriftServerSourceFactory.class)); } - } diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/zookeeper/TestMetricsZooKeeperSource.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/zookeeper/TestMetricsZooKeeperSource.java index 2ed502922e44..03674b55695c 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/zookeeper/TestMetricsZooKeeperSource.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/zookeeper/TestMetricsZooKeeperSource.java @@ -17,24 +17,22 @@ */ package org.apache.hadoop.hbase.zookeeper; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsZooKeeperSource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsZooKeeperSource.class); - - @Test(expected = RuntimeException.class) + @Test public void testGetInstanceNoHadoopCompat() throws Exception { // This should throw an exception because there is no compat lib on the class path. - CompatibilitySingletonFactory.getInstance(MetricsZooKeeperSource.class); + assertThrows(RuntimeException.class, + () -> CompatibilitySingletonFactory.getInstance(MetricsZooKeeperSource.class)); } } diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index 0c3d64509be0..2bab73cc723d 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -42,8 +42,18 @@ limitations under the License. hbase-hadoop-compat - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test @@ -132,11 +142,6 @@ limitations under the License. - - - org.apache.maven.plugins - maven-source-plugin - maven-assembly-plugin diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesAdapter.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesAdapter.java index 0d2c44c2e59a..6ae9c68edca1 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesAdapter.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesAdapter.java @@ -132,7 +132,9 @@ private void doRun() { } // register this as a MetricSource under different JMX Context'es. MetricsSourceAdapter adapter = new MetricsSourceAdapter(registry); - LOG.info("Registering " + info.getMetricsJmxContext() + " " + info.getMetricsDescription()); + if (LOG.isDebugEnabled()) { + LOG.debug("Registering {} {}", info.getMetricsJmxContext(), info.getMetricsDescription()); + } DefaultMetricsSystem.instance().register(info.getMetricsJmxContext(), info.getMetricsDescription(), adapter); registeredSources.put(info, adapter); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index e0429cfb55d1..dbd3b19850fa 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -369,7 +369,8 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { // rsWrap can be null because this function is called inside of init. if (rsWrap != null) { - addGaugesToMetricsRecordBuilder(mrb) + MetricsRecordBuilder metricsRecordBuilder = addGaugesToMetricsRecordBuilder(mrb); + metricsRecordBuilder .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC), rsWrap.getTotalRequestCount()) .addCounter( @@ -494,10 +495,17 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { rsWrap.getHedgedReadOpsInCurThread()) .addCounter(Interns.info(BLOCKED_REQUESTS_COUNT, BLOCKED_REQUESTS_COUNT_DESC), rsWrap.getBlockedRequestsCount()) + .addCounter(Interns.info(EXCLUDE_DATA_NODES_COUNT, EXCLUDE_DATA_NODES_COUNT_DESC), + rsWrap.getWALExcludeDNs().size()) .tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC), rsWrap.getZookeeperQuorum()) .tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), rsWrap.getServerName()) .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), rsWrap.getClusterId()); + if (!rsWrap.getWALExcludeDNs().isEmpty()) { + metricsRecordBuilder.tag( + Interns.info(EXCLUDE_DATA_NODES_DETAILS, EXCLUDE_DATA_NODES_DETAILS_DESC), + rsWrap.getWALExcludeDNs().toString()); + } } metricsRegistry.snapshot(mrb, all); @@ -560,6 +568,10 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions()) .addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC), rsWrap.getTotalBytesRead()) .addGauge(Interns.info(LOCAL_BYTES_READ, LOCAL_BYTES_READ_DESC), rsWrap.getLocalBytesRead()) + .addGauge(Interns.info(LOCAL_RACK_BYTES_READ, LOCAL_RACK_BYTES_READ_DESC), + rsWrap.getLocalRackBytesRead()) + .addGauge(Interns.info(REMOTE_RACK_BYTES_READ, REMOTE_RACK_BYTES_READ_DESC), + rsWrap.getRemoteRackBytesRead()) .addGauge(Interns.info(SHORTCIRCUIT_BYTES_READ, SHORTCIRCUIT_BYTES_READ_DESC), rsWrap.getShortCircuitBytesRead()) .addGauge(Interns.info(ZEROCOPY_BYTES_READ, ZEROCOPY_BYTES_READ_DESC), @@ -593,12 +605,20 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde rsWrap.getL1CacheEvictedCount()) .addGauge(Interns.info(L1_CACHE_HIT_COUNT, L1_CACHE_HIT_COUNT_DESC), rsWrap.getL1CacheHitCount()) + .addGauge(Interns.info(L1_CACHE_HIT_CAGHING_COUNT, L1_CACHE_HIT_CAGHING_COUNT_DESC), + rsWrap.getL1CacheHitCachingCount()) .addGauge(Interns.info(L1_CACHE_MISS_COUNT, L1_CACHE_MISS_COUNT_DESC), rsWrap.getL1CacheMissCount()) + .addGauge(Interns.info(L1_CACHE_MISS_CACHING_COUNT, L1_CACHE_MISS_CACHING_COUNT_DESC), + rsWrap.getL1CacheMissCachingCount()) .addGauge(Interns.info(L1_CACHE_HIT_RATIO, L1_CACHE_HIT_RATIO_DESC), rsWrap.getL1CacheHitRatio()) + .addGauge(Interns.info(L1_CACHE_HIT_CACHING_RATIO, L1_CACHE_HIT_CACHING_RATIO_DESC), + rsWrap.getL1CacheHitCachingRatio()) .addGauge(Interns.info(L1_CACHE_MISS_RATIO, L1_CACHE_MISS_RATIO_DESC), rsWrap.getL1CacheMissRatio()) + .addGauge(Interns.info(L1_CACHE_MISS_CACHING_RATIO, L1_CACHE_MISS_CACHING_RATIO_DESC), + rsWrap.getL1CacheMissCachingRatio()) .addGauge(Interns.info(L2_CACHE_SIZE, L2_CACHE_SIZE_DESC), rsWrap.getL2CacheSize()) .addGauge(Interns.info(L2_CACHE_FREE_SIZE, L2_CACHE_FREE_SIZE_DESC), rsWrap.getL2CacheFreeSize()) @@ -607,12 +627,20 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde rsWrap.getL2CacheEvictedCount()) .addGauge(Interns.info(L2_CACHE_HIT_COUNT, L2_CACHE_HIT_COUNT_DESC), rsWrap.getL2CacheHitCount()) + .addGauge(Interns.info(L2_CACHE_HIT_CACHING_COUNT, L2_CACHE_HIT_CACHING_COUNT_DESC), + rsWrap.getL2CacheHitCachingCount()) .addGauge(Interns.info(L2_CACHE_MISS_COUNT, L2_CACHE_MISS_COUNT_DESC), rsWrap.getL2CacheMissCount()) + .addGauge(Interns.info(L2_CACHE_MISS_CACHING_COUNT, L2_CACHE_MISS_CACHING_COUNT_DESC), + rsWrap.getL2CacheMissCachingCount()) .addGauge(Interns.info(L2_CACHE_HIT_RATIO, L2_CACHE_HIT_RATIO_DESC), rsWrap.getL2CacheHitRatio()) + .addGauge(Interns.info(L2_CACHE_HIT_CACHING_RATIO, L2_CACHE_HIT_CACHING_RATIO_DESC), + rsWrap.getL2CacheHitCachingRatio()) .addGauge(Interns.info(L2_CACHE_MISS_RATIO, L2_CACHE_MISS_RATIO_DESC), rsWrap.getL2CacheMissRatio()) + .addGauge(Interns.info(L2_CACHE_MISS_CACHING_RATIO, L2_CACHE_MISS_CACHING_RATIO_DESC), + rsWrap.getL2CacheMissCachingRatio()) .addGauge(Interns.info(MOB_FILE_CACHE_COUNT, MOB_FILE_CACHE_COUNT_DESC), rsWrap.getMobFileCacheCount()) .addGauge(Interns.info(MOB_FILE_CACHE_HIT_PERCENT, MOB_FILE_CACHE_HIT_PERCENT_DESC), diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 92ecaa580885..272b81190b07 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -215,6 +215,14 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) { this.regionWrapper.getMaxCompactedStoreFileRefCount()); mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE, MetricsRegionServerSource.MEMSTORE_SIZE_DESC), this.regionWrapper.getMemStoreSize()); + mrb.addGauge( + Interns.info(regionNamePrefix + MetricsRegionServerSource.MEMSTORE_HEAP_SIZE, + MetricsRegionServerSource.MEMSTORE_HEAP_SIZE_DESC), + this.regionWrapper.getMemStoreHeapSize()); + mrb.addGauge( + Interns.info(regionNamePrefix + MetricsRegionServerSource.MEMSTORE_OFFHEAP_SIZE, + MetricsRegionServerSource.MEMSTORE_OFFHEAP_SIZE_DESC), + this.regionWrapper.getMemStoreOffHeapSize()); mrb.addGauge( Interns.info(regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_AGE, MetricsRegionServerSource.MAX_STORE_FILE_AGE_DESC), @@ -284,6 +292,9 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) { MetricsRegionSource.ROW_READS_ONLY_ON_MEMSTORE_DESC); addCounter(mrb, this.regionWrapper.getMixedRowReadsCount(), MetricsRegionSource.MIXED_ROW_READS, MetricsRegionSource.MIXED_ROW_READS_ON_STORE_DESC); + mrb.add(Interns.tag(regionNamePrefix + MetricsRegionSource.TABLE_DESCRIPTOR_HASH, + MetricsRegionSource.TABLE_DESCRIPTOR_HASH_DESC, + this.regionWrapper.getTableDescriptorHash())); } } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java index 83577228f6dd..05e07e95ae6c 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java @@ -294,6 +294,14 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) { Interns.info(tableNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE, MetricsRegionServerSource.MEMSTORE_SIZE_DESC), tableWrapperAgg.getMemStoreSize(tableName.getNameAsString())); + mrb.addGauge( + Interns.info(tableNamePrefix + MetricsRegionServerSource.MEMSTORE_HEAP_SIZE, + MetricsRegionServerSource.MEMSTORE_HEAP_SIZE_DESC), + tableWrapperAgg.getMemStoreHeapSize(tableName.getNameAsString())); + mrb.addGauge( + Interns.info(tableNamePrefix + MetricsRegionServerSource.MEMSTORE_OFFHEAP_SIZE, + MetricsRegionServerSource.MEMSTORE_OFFHEAP_SIZE_DESC), + tableWrapperAgg.getMemStoreOffHeapSize(tableName.getNameAsString())); mrb.addGauge( Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_COUNT, MetricsRegionServerSource.STOREFILE_COUNT_DESC), diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterProcSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterProcSourceImpl.java index 04f95bf2eb91..26e7c1e26a57 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterProcSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterProcSourceImpl.java @@ -17,27 +17,22 @@ */ package org.apache.hadoop.hbase.master; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for MetricsMasterProcSourceImpl */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsMasterProcSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsMasterProcSourceImpl.class); - @Test public void testGetInstance() throws Exception { MetricsMasterProcSourceFactory metricsMasterProcSourceFactory = diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceImpl.java index 8dc1ae6e1636..becf1a4a7765 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceImpl.java @@ -17,25 +17,21 @@ */ package org.apache.hadoop.hbase.master; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for MetricsMasterSourceImpl */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsMasterSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsMasterSourceImpl.class); @Test public void testGetInstance() { diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java index 63f22143a91f..d8af6232aae6 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java @@ -17,32 +17,27 @@ */ package org.apache.hadoop.hbase.metrics; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test of default BaseSource for hadoop 2 */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestBaseSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBaseSourceImpl.class); - private static BaseSourceImpl bmsi; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { bmsi = new BaseSourceImpl("TestName", "test description", "testcontext", "TestContext"); } @@ -89,5 +84,4 @@ public void testRemoveMetric() throws Exception { bmsi.removeMetric("testrmgauge"); assertNull(bmsi.metricsRegistry.get("testrmgauge")); } - } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsExportHelper.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsExportHelper.java index 3a86dd049663..ff7436267e58 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsExportHelper.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsExportHelper.java @@ -18,25 +18,20 @@ package org.apache.hadoop.hbase.metrics; import java.util.Collection; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.impl.MetricsExportHelper; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsExportHelper { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsExportHelper.class); - @Test public void testExportHelper() { DefaultMetricsSystem.initialize("exportHelperTestSystem"); @@ -54,11 +49,11 @@ public void testExportHelper() { Collection metrics = MetricsExportHelper.export(); DefaultMetricsSystem.instance().stop(); - Assert.assertTrue(metrics.stream().anyMatch(mr -> mr.name().equals(metricsName))); - Assert.assertTrue(gaugeName + " is missing in the export", - contains(metrics, metricsName, gaugeName)); - Assert.assertTrue(counterName + " is missing in the export", - contains(metrics, metricsName, counterName)); + Assertions.assertTrue(metrics.stream().anyMatch(mr -> mr.name().equals(metricsName))); + Assertions.assertTrue(contains(metrics, metricsName, gaugeName), + gaugeName + " is missing in the export"); + Assertions.assertTrue(contains(metrics, metricsName, counterName), + counterName + " is missing in the export"); } private boolean contains(Collection metrics, String metricsName, diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java index 591bc9e63b5b..994fd92ede99 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java @@ -53,6 +53,16 @@ public long getMemStoreSize(String table) { return 1000; } + @Override + public long getMemStoreHeapSize(String table) { + return 1001; + } + + @Override + public long getMemStoreOffHeapSize(String table) { + return 1002; + } + @Override public long getStoreFileSize(String table) { return 2000; diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerSourceImpl.java index 3cda99b134eb..11bd183640c2 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerSourceImpl.java @@ -17,22 +17,19 @@ */ package org.apache.hadoop.hbase.regionserver; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsRegionServerSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsRegionServerSourceImpl.class); @Test public void testGetInstance() { @@ -44,10 +41,12 @@ public void testGetInstance() { CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceFactory.class)); } - @Test(expected = RuntimeException.class) + @Test public void testNoGetRegionServerMetricsSourceImpl() { // This should throw an exception because MetricsRegionServerSourceImpl should only // be created by a factory. - CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceImpl.class); + assertThrows(RuntimeException.class, () -> { + CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceImpl.class); + }); } } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index 2c8205085d1e..8f30f67e81d2 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -17,24 +17,21 @@ */ package org.apache.hadoop.hbase.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsRegionSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsRegionSourceImpl.class); @SuppressWarnings("SelfComparison") @Test @@ -56,11 +53,13 @@ public void testCompareToHashCodeEquals() { assertEquals(0, two.compareTo(two)); } - @Test(expected = RuntimeException.class) + @Test public void testNoGetRegionServerMetricsSourceImpl() { // This should throw an exception because MetricsRegionSourceImpl should only // be created by a factory. - CompatibilitySingletonFactory.getInstance(MetricsRegionSource.class); + assertThrows(RuntimeException.class, () -> { + CompatibilitySingletonFactory.getInstance(MetricsRegionSource.class); + }); } static class RegionWrapperStub implements MetricsRegionWrapper { @@ -111,6 +110,16 @@ public long getMemStoreSize() { return 0; } + @Override + public long getMemStoreHeapSize() { + return 0; + } + + @Override + public long getMemStoreOffHeapSize() { + return 0; + } + @Override public long getStoreFileSize() { return 0; @@ -232,5 +241,10 @@ public Map getMixedRowReadsCount() { map.put("info", 0L); return map; } + + @Override + public String getTableDescriptorHash() { + return "testhash"; + } } } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableSourceImpl.java index f1694801d7be..90e91939fee5 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableSourceImpl.java @@ -17,26 +17,23 @@ */ package org.apache.hadoop.hbase.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for MetricsTableSourceImpl */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsTableSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsTableSourceImpl.class); @SuppressWarnings("SelfComparison") @Test @@ -61,11 +58,13 @@ public void testCompareToHashCode() throws Exception { assertTrue(two.compareTo(two) == 0); } - @Test(expected = RuntimeException.class) + @Test public void testNoGetTableMetricsSourceImpl() { // This should throw an exception because MetricsTableSourceImpl should only // be created by a factory. - CompatibilitySingletonFactory.getInstance(MetricsTableSourceImpl.class); + assertThrows(RuntimeException.class, () -> { + CompatibilitySingletonFactory.getInstance(MetricsTableSourceImpl.class); + }); } @Test @@ -75,5 +74,4 @@ public void testGetTableMetrics() { .createTable("ONETABLE", new MetricsTableWrapperStub("ONETABLE")); assertEquals("ONETABLE", oneTbl.getTableName()); } - } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsUserSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsUserSourceImpl.java index b339dd8cc522..a9a392b74af2 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsUserSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsUserSourceImpl.java @@ -17,25 +17,21 @@ */ package org.apache.hadoop.hbase.regionserver; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsUserSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsUserSourceImpl.class); - @SuppressWarnings("SelfComparison") @Test public void testCompareToHashCodeEquals() throws Exception { @@ -56,11 +52,13 @@ public void testCompareToHashCodeEquals() throws Exception { assertTrue(two.compareTo(two) == 0); } - @Test(expected = RuntimeException.class) + @Test public void testNoGetRegionServerMetricsSourceImpl() throws Exception { // This should throw an exception because MetricsUserSourceImpl should only // be created by a factory. - CompatibilitySingletonFactory.getInstance(MetricsUserSource.class); + assertThrows(RuntimeException.class, () -> { + CompatibilitySingletonFactory.getInstance(MetricsUserSource.class); + }); } @Test diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWALSourceImpl.java index 300d536dc5f6..78a5e7eeb16d 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWALSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWALSourceImpl.java @@ -17,24 +17,19 @@ */ package org.apache.hadoop.hbase.regionserver.wal; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsWALSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsWALSourceImpl.class); - @Test public void testGetInstance() throws Exception { MetricsWALSource walSource = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java index c825d01b1628..07ac627eb678 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java @@ -17,21 +17,17 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsReplicationSourceFactoryImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsReplicationSourceFactoryImpl.class); @Test public void testGetInstance() { diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceImpl.java index 6c2284131049..7189ca2adc8c 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceImpl.java @@ -17,21 +17,17 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsReplicationSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsReplicationSourceImpl.class); @Test public void testGetInstance() throws Exception { diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/rest/TestMetricsRESTSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/rest/TestMetricsRESTSourceImpl.java index d1901f68bccc..5ee5c6aa2d25 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/rest/TestMetricsRESTSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/rest/TestMetricsRESTSourceImpl.java @@ -17,25 +17,21 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for hadoop 2's version of {@link MetricsRESTSource}. */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsRESTSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsRESTSourceImpl.class); @Test public void ensureCompatRegistered() { diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java index 7c7357c4f049..2978932dbc88 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashMap; import java.util.Locale; @@ -140,68 +140,68 @@ public void init() { public void assertTag(String name, String expected, BaseSource source) { getMetrics(source); String cName = canonicalizeMetricName(name); - assertEquals("Tags should be equal", expected, tags.get(cName)); + assertEquals(expected, tags.get(cName), "Tags should be equal"); } @Override public void assertGauge(String name, long expected, BaseSource source) { long found = getGaugeLong(name, source); - assertEquals("Metrics Should be equal", (long) Long.valueOf(expected), found); + assertEquals((long) Long.valueOf(expected), found, "Metrics Should be equal"); } @Override public void assertGaugeGt(String name, long expected, BaseSource source) { double found = getGaugeDouble(name, source); - assertTrue(name + " (" + found + ") should be greater than " + expected, found > expected); + assertTrue(found > expected, name + " (" + found + ") should be greater than " + expected); } @Override public void assertGaugeLt(String name, long expected, BaseSource source) { double found = getGaugeDouble(name, source); - assertTrue(name + "(" + found + ") should be less than " + expected, found < expected); + assertTrue(found < expected, name + "(" + found + ") should be less than " + expected); } @Override public void assertGauge(String name, double expected, BaseSource source) { double found = getGaugeDouble(name, source); - assertEquals("Metrics Should be equal", (double) Double.valueOf(expected), found, 0.01); + assertEquals((double) Double.valueOf(expected), found, 0.01, "Metrics Should be equal"); } @Override public void assertGaugeGt(String name, double expected, BaseSource source) { double found = getGaugeDouble(name, source); - assertTrue(name + "(" + found + ") should be greater than " + expected, found > expected); + assertTrue(found > expected, name + "(" + found + ") should be greater than " + expected); } @Override public void assertGaugeLt(String name, double expected, BaseSource source) { double found = getGaugeDouble(name, source); - assertTrue(name + "(" + found + ") should be less than " + expected, found < expected); + assertTrue(found < expected, name + "(" + found + ") should be less than " + expected); } @Override public void assertCounter(String name, long expected, BaseSource source) { long found = getCounter(name, source); - assertEquals(name + "(" + found + ") should be equal", (long) Long.valueOf(expected), found); + assertEquals((long) Long.valueOf(expected), found, name + "(" + found + ") should be equal"); } @Override public void assertCounterGt(String name, long expected, BaseSource source) { long found = getCounter(name, source); - assertTrue(name + " (" + found + ") should be greater than " + expected, found > expected); + assertTrue(found > expected, name + " (" + found + ") should be greater than " + expected); } @Override public void assertCounterLt(String name, long expected, BaseSource source) { long found = getCounter(name, source); - assertTrue(name + "(" + found + ") should be less than " + expected, found < expected); + assertTrue(found < expected, name + "(" + found + ") should be less than " + expected); } @Override public long getCounter(String name, BaseSource source) { getMetrics(source); String cName = canonicalizeMetricName(name); - assertNotNull("Should get counter " + cName + " but did not", counters.get(cName)); + assertNotNull(counters.get(cName), "Should get counter " + cName + " but did not"); return counters.get(cName).longValue(); } @@ -223,7 +223,7 @@ public boolean checkGaugeExists(String name, BaseSource source) { public double getGaugeDouble(String name, BaseSource source) { getMetrics(source); String cName = canonicalizeMetricName(name); - assertNotNull("Should get gauge " + cName + " but did not", gauges.get(cName)); + assertNotNull(gauges.get(cName), "Should get gauge " + cName + " but did not"); return gauges.get(cName).doubleValue(); } @@ -231,7 +231,7 @@ public double getGaugeDouble(String name, BaseSource source) { public long getGaugeLong(String name, BaseSource source) { getMetrics(source); String cName = canonicalizeMetricName(name); - assertNotNull("Should get gauge " + cName + " but did not", gauges.get(cName)); + assertNotNull(gauges.get(cName), "Should get gauge " + cName + " but did not"); return gauges.get(cName).longValue(); } @@ -252,7 +252,7 @@ private void reset() { private void getMetrics(BaseSource source) { reset(); if (!(source instanceof MetricsSource)) { - assertTrue("The Source passed must be a MetricsSource", false); + assertTrue(false, "The Source passed must be a MetricsSource"); } MetricsSource impl = (MetricsSource) source; diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/thrift/TestMetricsThriftServerSourceFactoryImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/thrift/TestMetricsThriftServerSourceFactoryImpl.java index 3bba46bcfa95..b2c7ca0672ee 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/thrift/TestMetricsThriftServerSourceFactoryImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/thrift/TestMetricsThriftServerSourceFactoryImpl.java @@ -17,33 +17,29 @@ */ package org.apache.hadoop.hbase.thrift; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for hadoop 2's version of MetricsThriftServerSourceFactory. */ -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsThriftServerSourceFactoryImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsThriftServerSourceFactoryImpl.class); @Test public void testCompatabilityRegistered() { assertNotNull( CompatibilitySingletonFactory.getInstance(MetricsThriftServerSourceFactory.class)); - assertTrue(CompatibilitySingletonFactory.getInstance( - MetricsThriftServerSourceFactory.class) instanceof MetricsThriftServerSourceFactoryImpl); + assertInstanceOf(MetricsThriftServerSourceFactoryImpl.class, + CompatibilitySingletonFactory.getInstance(MetricsThriftServerSourceFactory.class)); } @Test diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/zookeeper/TestMetricsZooKeeperSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/zookeeper/TestMetricsZooKeeperSourceImpl.java index bc200fd1e384..dd556931bc05 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/zookeeper/TestMetricsZooKeeperSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/zookeeper/TestMetricsZooKeeperSourceImpl.java @@ -17,28 +17,24 @@ */ package org.apache.hadoop.hbase.zookeeper; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMetricsZooKeeperSourceImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricsZooKeeperSourceImpl.class); @Test public void testGetInstance() { MetricsZooKeeperSource zkSource = CompatibilitySingletonFactory.getInstance(MetricsZooKeeperSource.class); - assertTrue(zkSource instanceof MetricsZooKeeperSourceImpl); + assertInstanceOf(MetricsZooKeeperSourceImpl.class, zkSource); assertSame(zkSource, CompatibilitySingletonFactory.getInstance(MetricsZooKeeperSource.class)); } } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableRangeHistogram.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableRangeHistogram.java index 62595a96bbda..19f0d0d15c07 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableRangeHistogram.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableRangeHistogram.java @@ -17,27 +17,22 @@ */ package org.apache.hadoop.metrics2.lib; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MetricsTests.class, SmallTests.class }) +@Tag(MetricsTests.TAG) +@Tag(SmallTests.TAG) public class TestMutableRangeHistogram { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMutableRangeHistogram.class); - private static final String RECORD_NAME = "test"; private static final String SIZE_HISTOGRAM_NAME = "TestSize"; diff --git a/hbase-hbtop/pom.xml b/hbase-hbtop/pom.xml index 370b17074f7f..53d73a1b8aa9 100644 --- a/hbase-hbtop/pom.xml +++ b/hbase-hbtop/pom.xml @@ -48,9 +48,24 @@ org.slf4j slf4j-api + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + org.mockito - mockito-core + mockito-junit-jupiter test diff --git a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/mode/RegionModeStrategy.java b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/mode/RegionModeStrategy.java index 5932ffb22791..02a6bd1f4643 100644 --- a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/mode/RegionModeStrategy.java +++ b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/mode/RegionModeStrategy.java @@ -112,7 +112,7 @@ private Record createRecord(ServerMetrics serverMetrics, RegionMetrics regionMet startKey = Bytes.toStringBinary(elements[1]); startCode = Bytes.toString(elements[2]); replicaId = - elements.length == 4 ? Integer.valueOf(Bytes.toString(elements[3])).toString() : ""; + elements.length == 4 ? Integer.valueOf(Bytes.toString(elements[3])).toString() : "0"; region = RegionInfo.encodeRegionName(regionMetrics.getRegionName()); } catch (IOException ignored) { // Exception deliberately ignored diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestRecord.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestRecord.java index 3790af32ed64..b76c2cff2d25 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestRecord.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestRecord.java @@ -21,19 +21,14 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestRecord { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestRecord.class); - @Test public void testBuilder() { Record actual1 = diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestRecordFilter.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestRecordFilter.java index 155b7942a66d..f069b049b6e0 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestRecordFilter.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestRecordFilter.java @@ -28,21 +28,15 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.Size; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestRecordFilter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRecordFilter.class); - @Test public void testParseAndBuilder() { testParseAndBuilder("REGION=region1", false, diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestUtils.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestUtils.java index 0968a1d5723f..6866337082e2 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestUtils.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/TestUtils.java @@ -19,7 +19,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import java.text.ParseException; import java.util.ArrayList; @@ -150,35 +150,35 @@ public static void assertRecordsInRegionMode(List records) { for (Record record : records) { switch (record.get(Field.REGION_NAME).asString()) { case "table1,,1.00000000000000000000000000000000.": - assertRecordInRegionMode(record, "default", "1", "", "table1", + assertRecordInRegionMode(record, "default", "1", "0", "table1", "00000000000000000000000000000000", "host1:1000", "host1.apache.com,1000,1", 0L, 0L, 0L, 0L, new Size(100, Size.Unit.MEGABYTE), new Size(200, Size.Unit.MEGABYTE), 1, new Size(100, Size.Unit.MEGABYTE), 0.1f, "", 100L, 100L, 100f, "2019-07-22 00:00:00"); break; case "table1,1,4.00000000000000000000000000000003.": - assertRecordInRegionMode(record, "default", "4", "", "table1", + assertRecordInRegionMode(record, "default", "4", "0", "table1", "00000000000000000000000000000003", "host2:1001", "host2.apache.com,1001,2", 0L, 0L, 0L, 0L, new Size(100, Size.Unit.MEGABYTE), new Size(200, Size.Unit.MEGABYTE), 1, new Size(100, Size.Unit.MEGABYTE), 0.4f, "1", 100L, 50L, 50f, "2019-07-22 00:00:03"); break; case "table2,,5.00000000000000000000000000000004.": - assertRecordInRegionMode(record, "default", "5", "", "table2", + assertRecordInRegionMode(record, "default", "5", "0", "table2", "00000000000000000000000000000004", "host2:1001", "host2.apache.com,1001,2", 0L, 0L, 0L, 0L, new Size(200, Size.Unit.MEGABYTE), new Size(400, Size.Unit.MEGABYTE), 2, new Size(200, Size.Unit.MEGABYTE), 0.5f, "", 200L, 150L, 75f, "2019-07-22 00:00:04"); break; case "table2,1,2.00000000000000000000000000000001.": - assertRecordInRegionMode(record, "default", "2", "", "table2", + assertRecordInRegionMode(record, "default", "2", "0", "table2", "00000000000000000000000000000001", "host1:1000", "host1.apache.com,1000,1", 0L, 0L, 0L, 0L, new Size(200, Size.Unit.MEGABYTE), new Size(400, Size.Unit.MEGABYTE), 2, new Size(200, Size.Unit.MEGABYTE), 0.2f, "1", 200L, 50L, 25f, "2019-07-22 00:00:01"); break; case "namespace:table3,,6.00000000000000000000000000000005.": - assertRecordInRegionMode(record, "namespace", "6", "", "table3", + assertRecordInRegionMode(record, "namespace", "6", "0", "table3", "00000000000000000000000000000005", "host2:1001", "host2.apache.com,1001,2", 0L, 0L, 0L, 0L, new Size(300, Size.Unit.MEGABYTE), new Size(600, Size.Unit.MEGABYTE), 3, new Size(300, Size.Unit.MEGABYTE), 0.6f, "", 300L, 200L, 66.66667f, diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/field/TestFieldValue.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/field/TestFieldValue.java index d2af864bdd02..eae630ddc93d 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/field/TestFieldValue.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/field/TestFieldValue.java @@ -19,22 +19,16 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.Size; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestFieldValue { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFieldValue.class); - @Test public void testParseAndAsSomethingMethod() { // String diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestClientMode.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestClientMode.java index a58033851401..2c863e27b8a2 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestClientMode.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestClientMode.java @@ -19,24 +19,18 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.TestUtils; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestClientMode extends TestModeBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestClientMode.class); - @Override protected Mode getMode() { return Mode.CLIENT; diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestModeBase.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestModeBase.java index 2d29fc414605..a170255f38ed 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestModeBase.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestModeBase.java @@ -20,7 +20,7 @@ import java.util.List; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.TestUtils; -import org.junit.Test; +import org.junit.jupiter.api.Test; public abstract class TestModeBase { diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestNamespaceMode.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestNamespaceMode.java index ab439fd826d9..c30aed7e4ece 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestNamespaceMode.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestNamespaceMode.java @@ -19,24 +19,18 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.TestUtils; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestNamespaceMode extends TestModeBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestNamespaceMode.class); - @Override protected Mode getMode() { return Mode.NAMESPACE; diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRegionMode.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRegionMode.java index f0756e48a952..6ff3d0ea5fbe 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRegionMode.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRegionMode.java @@ -22,20 +22,14 @@ import static org.hamcrest.MatcherAssert.assertThat; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.TestUtils; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestRegionMode extends TestModeBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionMode.class); - @Override protected Mode getMode() { return Mode.REGION; diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRegionServerMode.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRegionServerMode.java index 62cbeea5d139..19054aff8646 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRegionServerMode.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRegionServerMode.java @@ -19,24 +19,18 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.TestUtils; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestRegionServerMode extends TestModeBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionServerMode.class); - @Override protected Mode getMode() { return Mode.REGION_SERVER; diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRequestCountPerSecond.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRequestCountPerSecond.java index 25dca63d57ee..a00417ea22ab 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRequestCountPerSecond.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestRequestCountPerSecond.java @@ -20,19 +20,13 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestRequestCountPerSecond { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRequestCountPerSecond.class); - @Test public void test() { RequestCountPerSecond requestCountPerSecond = new RequestCountPerSecond(); diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestTableMode.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestTableMode.java index 0f05e484c1b2..19d219c78522 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestTableMode.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestTableMode.java @@ -19,24 +19,18 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.TestUtils; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestTableMode extends TestModeBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableMode.class); - @Override protected Mode getMode() { return Mode.TABLE; diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestUserMode.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestUserMode.java index 772e24a82c29..e7818d4edcd4 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestUserMode.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/mode/TestUserMode.java @@ -19,24 +19,18 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.TestUtils; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestUserMode extends TestModeBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestUserMode.class); - @Override protected Mode getMode() { return Mode.USER; diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/field/TestFieldScreenPresenter.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/field/TestFieldScreenPresenter.java index b6de23564016..348d338ac99e 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/field/TestFieldScreenPresenter.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/field/TestFieldScreenPresenter.java @@ -29,29 +29,23 @@ import java.util.EnumMap; import java.util.List; import java.util.stream.Collectors; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.hbtop.field.FieldInfo; import org.apache.hadoop.hbase.hbtop.mode.Mode; import org.apache.hadoop.hbase.hbtop.screen.top.TopScreenView; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InOrder; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@Category(SmallTests.class) -@RunWith(MockitoJUnitRunner.class) +@Tag(SmallTests.TAG) +@ExtendWith(MockitoExtension.class) public class TestFieldScreenPresenter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFieldScreenPresenter.class); - @Mock private FieldScreenView fieldScreenView; @@ -67,7 +61,7 @@ public class TestFieldScreenPresenter { private FieldScreenPresenter fieldScreenPresenter; - @Before + @BeforeEach public void setup() { Field sortField = Mode.REGION.getDefaultSortField(); fields = diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/help/TestHelpScreenPresenter.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/help/TestHelpScreenPresenter.java index 293f103e7d91..23368b98d608 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/help/TestHelpScreenPresenter.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/help/TestHelpScreenPresenter.java @@ -23,25 +23,19 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.screen.top.TopScreenView; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@Category(SmallTests.class) -@RunWith(MockitoJUnitRunner.class) +@Tag(SmallTests.TAG) +@ExtendWith(MockitoExtension.class) public class TestHelpScreenPresenter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHelpScreenPresenter.class); - private static final long TEST_REFRESH_DELAY = 5; @Mock @@ -52,7 +46,7 @@ public class TestHelpScreenPresenter { private HelpScreenPresenter helpScreenPresenter; - @Before + @BeforeEach public void setup() { helpScreenPresenter = new HelpScreenPresenter(helpScreenView, TEST_REFRESH_DELAY, topScreenView); diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/mode/TestModeScreenPresenter.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/mode/TestModeScreenPresenter.java index 8202f5d64cb4..5676836e4092 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/mode/TestModeScreenPresenter.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/mode/TestModeScreenPresenter.java @@ -26,25 +26,19 @@ import java.util.Arrays; import java.util.function.Consumer; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.mode.Mode; import org.apache.hadoop.hbase.hbtop.screen.top.TopScreenView; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@Category(SmallTests.class) -@RunWith(MockitoJUnitRunner.class) +@Tag(SmallTests.TAG) +@ExtendWith(MockitoExtension.class) public class TestModeScreenPresenter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestModeScreenPresenter.class); - @Mock private ModeScreenView modeScreenView; diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestFilterDisplayModeScreenPresenter.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestFilterDisplayModeScreenPresenter.java index df1a03a5ad67..f8905d92996b 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestFilterDisplayModeScreenPresenter.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestFilterDisplayModeScreenPresenter.java @@ -25,28 +25,22 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.RecordFilter; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.hbtop.field.FieldInfo; import org.apache.hadoop.hbase.hbtop.mode.Mode; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@Category(SmallTests.class) -@RunWith(MockitoJUnitRunner.class) +@Tag(SmallTests.TAG) +@ExtendWith(MockitoExtension.class) public class TestFilterDisplayModeScreenPresenter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFilterDisplayModeScreenPresenter.class); - @Mock private FilterDisplayModeScreenView filterDisplayModeScreenView; @@ -55,7 +49,7 @@ public class TestFilterDisplayModeScreenPresenter { private FilterDisplayModeScreenPresenter filterDisplayModeScreenPresenter; - @Before + @BeforeEach public void setup() { List fields = Mode.REGION.getFieldInfos().stream().map(FieldInfo::getField).collect(Collectors.toList()); diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestInputModeScreenPresenter.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestInputModeScreenPresenter.java index d243e214562f..8f93a08084c2 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestInputModeScreenPresenter.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestInputModeScreenPresenter.java @@ -28,26 +28,20 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Function; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.screen.ScreenView; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InOrder; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@Category(SmallTests.class) -@RunWith(MockitoJUnitRunner.class) +@Tag(SmallTests.TAG) +@ExtendWith(MockitoExtension.class) public class TestInputModeScreenPresenter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestInputModeScreenPresenter.class); - private static final String TEST_INPUT_MESSAGE = "test input message"; @Mock @@ -61,7 +55,7 @@ public class TestInputModeScreenPresenter { private InputModeScreenPresenter inputModeScreenPresenter; - @Before + @BeforeEach public void setup() { List histories = new ArrayList<>(); histories.add("history1"); diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestMessageModeScreenPresenter.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestMessageModeScreenPresenter.java index 823b9b485d03..9e181533b49f 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestMessageModeScreenPresenter.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestMessageModeScreenPresenter.java @@ -22,24 +22,18 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@Category(SmallTests.class) -@RunWith(MockitoJUnitRunner.class) +@Tag(SmallTests.TAG) +@ExtendWith(MockitoExtension.class) public class TestMessageModeScreenPresenter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMessageModeScreenPresenter.class); - private static final String TEST_MESSAGE = "test message"; @Mock @@ -50,7 +44,7 @@ public class TestMessageModeScreenPresenter { private MessageModeScreenPresenter messageModeScreenPresenter; - @Before + @BeforeEach public void setup() { messageModeScreenPresenter = new MessageModeScreenPresenter(messageModeScreenView, TEST_MESSAGE, topScreenView); diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestPaging.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestPaging.java index f5a90cc6071f..f3e5ffbf374b 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestPaging.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestPaging.java @@ -20,18 +20,13 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestPaging { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestPaging.class); - @Test public void testArrowUpAndArrowDown() { Paging paging = new Paging(); diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestTopScreenModel.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestTopScreenModel.java index c44d1e6b1f7f..fb04ccc0e905 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestTopScreenModel.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestTopScreenModel.java @@ -19,14 +19,13 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.when; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.lenient; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.RecordFilter; @@ -36,22 +35,17 @@ import org.apache.hadoop.hbase.hbtop.field.FieldValue; import org.apache.hadoop.hbase.hbtop.mode.Mode; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@Category(SmallTests.class) -@RunWith(MockitoJUnitRunner.class) +@Tag(SmallTests.TAG) +@ExtendWith(MockitoExtension.class) public class TestTopScreenModel { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTopScreenModel.class); - @Mock private Admin admin; @@ -59,9 +53,9 @@ public class TestTopScreenModel { private List fields; - @Before + @BeforeEach public void setup() throws IOException { - when(admin.getClusterMetrics()).thenReturn(TestUtils.createDummyClusterMetrics()); + lenient().when(admin.getClusterMetrics()).thenReturn(TestUtils.createDummyClusterMetrics()); topScreenModel = new TopScreenModel(admin, Mode.REGION, null, null, null, null); fields = diff --git a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestTopScreenPresenter.java b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestTopScreenPresenter.java index e633303a111a..9f75ed4cd348 100644 --- a/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestTopScreenPresenter.java +++ b/hbase-hbtop/src/test/java/org/apache/hadoop/hbase/hbtop/screen/top/TestTopScreenPresenter.java @@ -27,29 +27,23 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.hbtop.Record; import org.apache.hadoop.hbase.hbtop.field.Field; import org.apache.hadoop.hbase.hbtop.field.FieldInfo; import org.apache.hadoop.hbase.hbtop.terminal.TerminalSize; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InOrder; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@Category(SmallTests.class) -@RunWith(MockitoJUnitRunner.class) +@Tag(SmallTests.TAG) +@ExtendWith(MockitoExtension.class) public class TestTopScreenPresenter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTopScreenPresenter.class); - private static final List TEST_FIELD_INFOS = Arrays.asList( new FieldInfo(Field.REGION, 10, true), new FieldInfo(Field.REQUEST_COUNT_PER_SECOND, 10, true), new FieldInfo(Field.LOCALITY, 10, true)); @@ -73,7 +67,7 @@ public class TestTopScreenPresenter { private TopScreenPresenter topScreenPresenter; - @Before + @BeforeEach public void setup() { when(topScreenView.getTerminalSize()).thenReturn(new TerminalSize(100, 100)); when(topScreenView.getPageSize()).thenReturn(100); diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 47373b25e779..1774f29b9c5f 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -105,6 +105,11 @@ bcprov-jdk18on test + + org.bouncycastle + bcpkix-jdk18on + test + org.apache.httpcomponents httpclient @@ -116,8 +121,18 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test @@ -193,13 +208,13 @@ false - ${build.year} - ${license.debug.print.included} - ${license.bundles.dependencies} - ${license.bundles.jquery} - ${license.bundles.vega} - ${license.bundles.logo} - ${license.bundles.bootstrap} + ${build.year} + ${license.debug.print.included} + ${license.bundles.dependencies} + ${license.bundles.jquery} + ${license.bundles.vega} + ${license.bundles.logo} + ${license.bundles.bootstrap} ${project.groupId}:hbase-resource-bundle:${project.version} @@ -224,20 +239,6 @@ true - - - org.apache.maven.plugins - maven-source-plugin - - - - jar - test-jar - - package - - - maven-surefire-plugin @@ -471,6 +472,10 @@ org.bouncycastle bcprov-jdk15on + + org.bouncycastle + bcpkix-jdk15on + diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index 36a101b6ac73..6012b24ec543 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -228,7 +228,10 @@ public static class Builder { private String usernameConfKey; private String keytabConfKey; private boolean needsClientAuth; + private String includeCiphers; private String excludeCiphers; + private String includeProtocols; + private String excludeProtocols; private String hostName; private String appDir = APP_DIR; @@ -401,10 +404,32 @@ public Builder setLogDir(String logDir) { return this; } + @Deprecated + // Use setExcludeCiphers() which supports the fluent builder API public void excludeCiphers(String excludeCiphers) { this.excludeCiphers = excludeCiphers; } + public Builder setExcludeCiphers(String excludeCiphers) { + this.excludeCiphers = excludeCiphers; + return this; + } + + public Builder setIncludeCiphers(String includeCiphers) { + this.includeCiphers = includeCiphers; + return this; + } + + public Builder setIncludeProtocols(String includeProtocols) { + this.includeProtocols = includeProtocols; + return this; + } + + public Builder setExcludeProtocols(String excludeProtocols) { + this.excludeProtocols = excludeProtocols; + return this; + } + public HttpServer build() throws IOException { // Do we still need to assert this non null name if it is deprecated? @@ -466,6 +491,21 @@ public HttpServer build() throws IOException { sslCtxFactory.setTrustStorePassword(trustStorePassword); } + if (includeProtocols != null && !includeProtocols.trim().isEmpty()) { + sslCtxFactory.setIncludeProtocols(StringUtils.getTrimmedStrings(includeProtocols)); + LOG.debug("Included TLS Protocol List:" + includeProtocols); + } + + if (excludeProtocols != null && !excludeProtocols.trim().isEmpty()) { + sslCtxFactory.setExcludeProtocols(StringUtils.getTrimmedStrings(excludeProtocols)); + LOG.debug("Excluded TLS Protocol List:" + excludeProtocols); + } + + if (includeCiphers != null && !includeCiphers.trim().isEmpty()) { + sslCtxFactory.setIncludeCipherSuites(StringUtils.getTrimmedStrings(includeCiphers)); + LOG.debug("Included SSL Cipher List:" + includeCiphers); + } + if (excludeCiphers != null && !excludeCiphers.trim().isEmpty()) { sslCtxFactory.setExcludeCipherSuites(StringUtils.getTrimmedStrings(excludeCiphers)); LOG.debug("Excluded SSL Cipher List:" + excludeCiphers); diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java index aa25ef427629..ea73be808f07 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java @@ -42,6 +42,9 @@ public class InfoServer { private static final String HBASE_APP_DIR = "hbase-webapps"; private final org.apache.hadoop.hbase.http.HttpServer httpServer; + private static final String HADOOP_WEB_TLS_CONFIG_PREFIX = "ssl.server."; + private static final String HBASE_WEB_TLS_CONFIG_PREFIX = "hbase.ui.ssl."; + /** * Create a status server on the given port. The jsp scripts are taken from * src/hbase-webapps/name. @@ -67,15 +70,19 @@ public InfoServer(String name, String bindAddress, int port, boolean findPort, builder.setLogDir(logDir); } if (httpConfig.isSecure()) { - builder - .keyPassword(HBaseConfiguration.getPassword(c, "ssl.server.keystore.keypassword", null)) - .keyStore(c.get("ssl.server.keystore.location"), - HBaseConfiguration.getPassword(c, "ssl.server.keystore.password", null), - c.get("ssl.server.keystore.type", "jks")) - .trustStore(c.get("ssl.server.truststore.location"), - HBaseConfiguration.getPassword(c, "ssl.server.truststore.password", null), - c.get("ssl.server.truststore.type", "jks")); - builder.excludeCiphers(c.get("ssl.server.exclude.cipher.list")); + // We are using the Hadoop HTTP server config properties. + // This makes it easy to keep in sync with Hadoop's UI servers, but hard to set this + // separately for HBase. + builder.keyPassword(getTLSPassword(c, "keystore.keypassword")) + .keyStore(getTLSProperty(c, "keystore.location"), getTLSPassword(c, "keystore.password"), + getTLSProperty(c, "keystore.type", "jks")) + .trustStore(getTLSProperty(c, "truststore.location"), + getTLSPassword(c, "truststore.password"), getTLSProperty(c, "truststore.type", "jks")) + // The ssl.server.*.protocols properties do not exist in Hadoop at the time of writing. + .setIncludeProtocols(getTLSProperty(c, "include.protocols")) + .setExcludeProtocols(getTLSProperty(c, "exclude.protocols")) + .setIncludeCiphers(getTLSProperty(c, "include.cipher.list")) + .setExcludeCiphers(getTLSProperty(c, "exclude.cipher.list")); } final String httpAuthType = c.get(HttpServer.HTTP_UI_AUTHENTICATION, "").toLowerCase(); @@ -97,6 +104,20 @@ public InfoServer(String name, String bindAddress, int port, boolean findPort, this.httpServer = builder.build(); } + private String getTLSPassword(Configuration c, String postfix) throws IOException { + return HBaseConfiguration.getPassword(c, HBASE_WEB_TLS_CONFIG_PREFIX + postfix, + HBaseConfiguration.getPassword(c, HADOOP_WEB_TLS_CONFIG_PREFIX + postfix, null)); + } + + private String getTLSProperty(Configuration c, String postfix) { + return getTLSProperty(c, postfix, null); + } + + private String getTLSProperty(Configuration c, String postfix, String defaultValue) { + return c.get(HBASE_WEB_TLS_CONFIG_PREFIX + postfix, + c.get(HADOOP_WEB_TLS_CONFIG_PREFIX + postfix, defaultValue)); + } + /** * Builds an ACL that will restrict the users who can issue commands to endpoints on the UI which * are meant only for administrators. diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java index a9e8fa7cbe16..1a23cf46076b 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java @@ -27,6 +27,7 @@ import java.nio.charset.StandardCharsets; import java.util.Objects; import java.util.regex.Pattern; +import java.util.stream.Collectors; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; import javax.servlet.ServletException; @@ -61,6 +62,7 @@ public final class LogLevel { public static final String PROTOCOL_HTTPS = "https"; public static final String READONLY_LOGGERS_CONF_KEY = "hbase.ui.logLevels.readonly.loggers"; + public static final String MASTER_UI_READONLY_CONF_KEY = "hbase.master.ui.readonly"; /** * A command line implementation @@ -214,7 +216,11 @@ private int parseProtocolArgs(String[] args, int index) throws HadoopIllegalArgu * @throws Exception if unable to connect */ private void doGetLevel() throws Exception { - process(protocol + "://" + hostName + "/logLevel?log=" + className); + System.out.println(fetchGetLevelResponse()); + } + + String fetchGetLevelResponse() throws Exception { + return fetchResponse(protocol + "://" + hostName + "/logLevel?log=" + className); } /** @@ -223,7 +229,12 @@ private void doGetLevel() throws Exception { * @throws Exception if unable to connect */ private void doSetLevel() throws Exception { - process(protocol + "://" + hostName + "/logLevel?log=" + className + "&level=" + level); + System.out.println(fetchSetLevelResponse()); + } + + String fetchSetLevelResponse() throws Exception { + return fetchResponse( + protocol + "://" + hostName + "/logLevel?log=" + className + "&level=" + level); } /** @@ -257,11 +268,12 @@ private HttpURLConnection connect(URL url) throws Exception { } /** - * Configures the client to send HTTP request to the URL. Supports SPENGO for authentication. + * Send HTTP request and fetch response. * @param urlString URL and query string to the daemon's web UI + * @return the response from the daemon * @throws Exception if unable to connect */ - private void process(String urlString) throws Exception { + private String fetchResponse(String urlString) throws Exception { URL url = new URL(urlString); System.out.println("Connecting to " + url); @@ -269,16 +281,14 @@ private void process(String urlString) throws Exception { HttpExceptionUtils.validateResponse(connection, 200); - // read from the servlet - try ( InputStreamReader streamReader = new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8); BufferedReader bufferedReader = new BufferedReader(streamReader)) { - bufferedReader.lines().filter(Objects::nonNull).filter(line -> line.startsWith(MARKER)) - .forEach(line -> System.out.println(TAG.matcher(line).replaceAll(""))); - } catch (IOException ioe) { - System.err.println("" + ioe); + + return bufferedReader.lines().filter(Objects::nonNull) + .filter(line -> line.startsWith(MARKER)).map(line -> TAG.matcher(line).replaceAll("")) + .collect(Collectors.joining("\n")); } } } @@ -300,14 +310,6 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) if (!HttpServer.hasAdministratorAccess(getServletContext(), request, response)) { return; } - // Disallow modification of the LogLevel if explicitly set to readonly - Configuration conf = - (Configuration) getServletContext().getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE); - if (conf.getBoolean("hbase.master.ui.readonly", false)) { - sendError(response, HttpServletResponse.SC_FORBIDDEN, - "Modification of HBase via the UI is disallowed in configuration."); - return; - } response.setContentType("text/html"); PrintWriter out; try { @@ -323,6 +325,8 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) String logName = ServletUtil.getParameter(request, "log"); String level = ServletUtil.getParameter(request, "level"); + Configuration conf = + (Configuration) getServletContext().getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE); String[] readOnlyLogLevels = conf.getStrings(READONLY_LOGGERS_CONF_KEY); if (logName != null) { @@ -332,6 +336,13 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) Logger log = LoggerFactory.getLogger(logName); out.println(MARKER + "Log Class: " + log.getClass().getName() + "
"); if (level != null) { + // Disallow modification of the LogLevel if explicitly set to readonly + if (conf.getBoolean(MASTER_UI_READONLY_CONF_KEY, false)) { + sendError(response, HttpServletResponse.SC_FORBIDDEN, + "Modification of HBase via the UI is disallowed in configuration."); + return; + } + if (!isLogLevelChangeAllowed(logName, readOnlyLogLevels)) { sendError(response, HttpServletResponse.SC_PRECONDITION_FAILED, "Modification of logger " + logName + " is disallowed in configuration."); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/HttpServerFunctionalTest.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/HttpServerFunctionalTest.java index c11a203620ad..c7b8db2cb0b4 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/HttpServerFunctionalTest.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/HttpServerFunctionalTest.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; import java.io.BufferedReader; import java.io.File; @@ -215,7 +215,7 @@ public static void stop(HttpServer server) throws Exception { * @throws MalformedURLException if the URL cannot be created. */ public static URL getServerURL(HttpServer server) throws MalformedURLException { - assertNotNull("No server", server); + assertNotNull(server, "No server"); return new URL("http://" + NetUtils.getHostPortString(server.getConnectorAddress(0))); } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapServerTestBase.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapServerTestBase.java index bbf35b8585f6..8856aaa0e205 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapServerTestBase.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapServerTestBase.java @@ -21,34 +21,73 @@ import java.net.HttpURLConnection; import java.net.URL; import org.apache.commons.codec.binary.Base64; -import org.apache.directory.server.core.integ.CreateLdapServerRule; +import org.apache.directory.ldap.client.template.LdapConnectionTemplate; +import org.apache.directory.server.core.api.DirectoryService; +import org.apache.directory.server.core.integ.ApacheDSTestExtension; +import org.apache.directory.server.ldap.LdapServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.http.resource.JerseyResource; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Base class for setting up and testing an HTTP server with LDAP authentication. */ +@ExtendWith(ApacheDSTestExtension.class) public class LdapServerTestBase extends HttpServerFunctionalTest { private static final Logger LOG = LoggerFactory.getLogger(LdapServerTestBase.class); - @ClassRule - public static CreateLdapServerRule ldapRule = new CreateLdapServerRule(); - protected static HttpServer server; protected static URL baseUrl; + /** + * The following fields are set by ApacheDSTestExtension. These are normally inherited from + * AbstractLdapTestUnit, but this class already has a parent. We only use ldapServer, but + * declaring that one alone does not work. + */ + + /** The class DirectoryService instance */ + public static DirectoryService classDirectoryService; + + /** The test DirectoryService instance */ + public static DirectoryService methodDirectoryService; + + /** The current DirectoryService instance */ + public static DirectoryService directoryService; + + /** The class LdapServer instance */ + public static LdapServer classLdapServer; + + /** The test LdapServer instance */ + public static LdapServer methodLdapServer; + + /** The current LdapServer instance */ + public static LdapServer ldapServer; + + /** The Ldap connection template */ + public static LdapConnectionTemplate ldapConnectionTemplate; + + /** The current revision */ + public static long revision = 0L; + + /** + * End of fields required by ApacheDSTestExtension + */ + private static final String AUTH_TYPE = "Basic "; + protected static LdapServer getLdapServer() { + return classLdapServer; + } + /** * Sets up the HTTP server with LDAP authentication before any tests are run. * @throws Exception if an error occurs during server setup */ - @BeforeClass + @BeforeAll public static void setupServer() throws Exception { Configuration conf = new Configuration(); setLdapConfigurations(conf); @@ -66,7 +105,7 @@ public static void setupServer() throws Exception { * Stops the HTTP server after all tests are completed. * @throws Exception if an error occurs during server shutdown */ - @AfterClass + @AfterAll public static void stopServer() throws Exception { try { if (null != server) { @@ -90,8 +129,8 @@ protected static void setLdapConfigurations(Configuration conf) { conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY, "org.apache.hadoop.hbase.http.lib.AuthenticationFilterInitializer"); conf.set("hadoop.http.authentication.type", "ldap"); - conf.set("hadoop.http.authentication.ldap.providerurl", String.format("ldap://%s:%s", - LdapConstants.LDAP_SERVER_ADDR, ldapRule.getLdapServer().getPort())); + conf.set("hadoop.http.authentication.ldap.providerurl", + String.format("ldap://%s:%s", LdapConstants.LDAP_SERVER_ADDR, getLdapServer().getPort())); conf.set("hadoop.http.authentication.ldap.enablestarttls", "false"); conf.set("hadoop.http.authentication.ldap.basedn", LdapConstants.LDAP_BASE_DN); } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java index 80b02006c036..46e8902bbd61 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Set; @@ -30,21 +30,17 @@ import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.net.NetUtils; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestGlobalFilter extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestGlobalFilter.class); private static final Logger LOG = LoggerFactory.getLogger(HttpServer.class); private static final Set RECORDS = new TreeSet<>(); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHtmlQuoting.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHtmlQuoting.java index 0fe46194083a..4728e1f057e3 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHtmlQuoting.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHtmlQuoting.java @@ -17,26 +17,22 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import javax.servlet.http.HttpServletRequest; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestHtmlQuoting { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHtmlQuoting.class); @Test public void testNeedsQuoting() throws Exception { @@ -85,17 +81,17 @@ public void testRequestQuoting() throws Exception { new HttpServer.QuotingInputFilter.RequestQuoter(mockReq); Mockito.doReturn("a cookies = HttpCookie.parse(header); - Assert.assertTrue(!cookies.isEmpty()); - Assert.assertTrue(header.contains("; HttpOnly")); - Assert.assertTrue("token".equals(cookies.get(0).getValue())); + assertTrue(!cookies.isEmpty()); + assertTrue(header.contains("; HttpOnly")); + assertTrue("token".equals(cookies.get(0).getValue())); } @Test @@ -141,13 +138,13 @@ public void testHttpsCookie() throws IOException, GeneralSecurityException { conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory()); String header = conn.getHeaderField("Set-Cookie"); - Assert.assertTrue(header != null); + assertTrue(header != null); List cookies = HttpCookie.parse(header); - Assert.assertTrue(!cookies.isEmpty()); - Assert.assertTrue(header.contains("; HttpOnly")); - Assert.assertTrue(cookies.get(0).getSecure()); - Assert.assertTrue("token".equals(cookies.get(0).getValue())); + assertTrue(!cookies.isEmpty()); + assertTrue(header.contains("; HttpOnly")); + assertTrue(cookies.get(0).getSecure()); + assertTrue("token".equals(cookies.get(0).getValue())); } @Test @@ -162,16 +159,16 @@ public void testHttpsCookieDefaultServlets() throws Exception { conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory()); String header = conn.getHeaderField("Set-Cookie"); - Assert.assertTrue(header != null); + assertTrue(header != null); List cookies = HttpCookie.parse(header); - Assert.assertTrue(!cookies.isEmpty()); - Assert.assertTrue(header.contains("; HttpOnly")); - Assert.assertTrue(cookies.get(0).getSecure()); - Assert.assertTrue("token".equals(cookies.get(0).getValue())); + assertTrue(!cookies.isEmpty()); + assertTrue(header.contains("; HttpOnly")); + assertTrue(cookies.get(0).getSecure()); + assertTrue("token".equals(cookies.get(0).getValue())); } } - @AfterClass + @AfterAll public static void cleanup() throws Exception { server.stop(); FileUtil.fullyDelete(new File(BASEDIR)); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLog.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLog.java index 904b1d4ca0ad..2e5bffacf855 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLog.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLog.java @@ -19,31 +19,26 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.CustomRequestLog; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.RequestLog; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLogWriter; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestHttpRequestLog { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHttpRequestLog.class); - @Test public void testAppenderDefined() { RequestLog requestLog = HttpRequestLog.getRequestLog("test"); - assertNotNull("RequestLog should not be null", requestLog); + assertNotNull(requestLog, "RequestLog should not be null"); assertThat(requestLog, instanceOf(CustomRequestLog.class)); CustomRequestLog crl = (CustomRequestLog) requestLog; assertThat(crl.getWriter(), instanceOf(Slf4jRequestLogWriter.class)); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java index ad9c9d3a0671..7091a2ac7014 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java @@ -18,10 +18,11 @@ package org.apache.hadoop.hbase.http; import static org.hamcrest.Matchers.greaterThan; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedReader; import java.io.IOException; @@ -57,7 +58,6 @@ import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.http.HttpServer.QuotingInputFilter.RequestQuoter; import org.apache.hadoop.hbase.http.resource.JerseyResource; import org.apache.hadoop.hbase.testclassification.MiscTests; @@ -74,13 +74,11 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.hamcrest.MatcherAssert; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,11 +86,9 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ajax.JSON; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestHttpServer extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHttpServer.class); private static final Logger LOG = LoggerFactory.getLogger(TestHttpServer.class); private static HttpServer server; @@ -148,7 +144,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro public static class LongHeaderServlet extends HttpServlet { @Override public void doGet(HttpServletRequest request, HttpServletResponse response) { - Assert.assertEquals(63 * 1024, request.getHeader("longheader").length()); + assertEquals(63 * 1024, request.getHeader("longheader").length()); response.setStatus(HttpServletResponse.SC_OK); } } @@ -164,7 +160,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro } } - @BeforeClass + @BeforeAll public static void setup() throws Exception { Configuration conf = new Configuration(); conf.setInt(HttpServer.HTTP_MAX_THREADS, MAX_THREADS); @@ -179,7 +175,7 @@ public static void setup() throws Exception { LOG.info("HTTP server started: " + baseUrl); } - @AfterClass + @AfterAll public static void cleanup() throws Exception { server.stop(); } @@ -201,9 +197,8 @@ public void testMaxThreads() throws Exception { start.await(); assertEquals("a:b\nc:d\n", readOutput(new URL(baseUrl, "/echo?a=b&c=d"))); int serverThreads = server.webServer.getThreadPool().getThreads(); - assertTrue( - "More threads are started than expected, Server Threads count: " + serverThreads, - serverThreads <= MAX_THREADS); + assertTrue(serverThreads <= MAX_THREADS, + "More threads are started than expected, Server Threads count: " + serverThreads); LOG.info("Number of threads = " + serverThreads + " which is less or equal than the max = " + MAX_THREADS); } catch (Exception e) { @@ -419,7 +414,7 @@ public List getGroups(String user) { * authentication filters are set, but authorization is not enabled. */ @Test - @Ignore + @Disabled public void testDisabledAuthorizationOfDefaultServlets() throws Exception { Configuration conf = new Configuration(); @@ -450,7 +445,7 @@ public void testDisabledAuthorizationOfDefaultServlets() throws Exception { * Verify the administrator access for /logs, /stacks, /conf, /logLevel and /metrics servlets. */ @Test - @Ignore + @Disabled public void testAuthorizationOfDefaultServlets() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true); @@ -491,8 +486,8 @@ public void testRequestQuoterWithNull() { Mockito.doReturn(null).when(request).getParameterValues("dummy"); RequestQuoter requestQuoter = new RequestQuoter(request); String[] parameterValues = requestQuoter.getParameterValues("dummy"); - Assert.assertNull("It should return null " + "when there are no values for the parameter", - parameterValues); + assertNull(parameterValues, + "It should return null " + "when there are no values for the parameter"); } @Test @@ -502,7 +497,7 @@ public void testRequestQuoterWithNotNull() { Mockito.doReturn(values).when(request).getParameterValues("dummy"); RequestQuoter requestQuoter = new RequestQuoter(request); String[] parameterValues = requestQuoter.getParameterValues("dummy"); - Assert.assertTrue("It should return Parameter Values", Arrays.equals(values, parameterValues)); + assertTrue(Arrays.equals(values, parameterValues), "It should return Parameter Values"); } @SuppressWarnings("unchecked") @@ -533,26 +528,26 @@ public void testHasAdministratorAccess() throws Exception { HttpServletResponse response = Mockito.mock(HttpServletResponse.class); // authorization OFF - Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response)); + assertTrue(HttpServer.hasAdministratorAccess(context, request, response)); // authorization ON & user NULL response = Mockito.mock(HttpServletResponse.class); conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true); - Assert.assertFalse(HttpServer.hasAdministratorAccess(context, request, response)); + assertFalse(HttpServer.hasAdministratorAccess(context, request, response)); Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString()); // authorization ON & user NOT NULL & ACLs NULL response = Mockito.mock(HttpServletResponse.class); Mockito.when(request.getRemoteUser()).thenReturn("foo"); - Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response)); + assertTrue(HttpServer.hasAdministratorAccess(context, request, response)); // authorization ON & user NOT NULL & ACLs NOT NULL & user not in ACLs response = Mockito.mock(HttpServletResponse.class); AccessControlList acls = Mockito.mock(AccessControlList.class); Mockito.when(acls.isUserAllowed(Mockito. any())).thenReturn(false); Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls); - Assert.assertFalse(HttpServer.hasAdministratorAccess(context, request, response)); + assertFalse(HttpServer.hasAdministratorAccess(context, request, response)); Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_FORBIDDEN), Mockito.anyString()); @@ -560,7 +555,7 @@ public void testHasAdministratorAccess() throws Exception { response = Mockito.mock(HttpServletResponse.class); Mockito.when(acls.isUserAllowed(Mockito. any())).thenReturn(true); Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls); - Assert.assertTrue(HttpServer.hasAdministratorAccess(context, request, response)); + assertTrue(HttpServer.hasAdministratorAccess(context, request, response)); } @@ -573,7 +568,7 @@ public void testRequiresAuthorizationAccess() throws Exception { HttpServletResponse response = Mockito.mock(HttpServletResponse.class); // requires admin access to instrumentation, FALSE by default - Assert.assertTrue(HttpServer.isInstrumentationAccessAllowed(context, request, response)); + assertTrue(HttpServer.isInstrumentationAccessAllowed(context, request, response)); // requires admin access to instrumentation, TRUE conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN, true); @@ -581,7 +576,7 @@ public void testRequiresAuthorizationAccess() throws Exception { AccessControlList acls = Mockito.mock(AccessControlList.class); Mockito.when(acls.isUserAllowed(Mockito. any())).thenReturn(false); Mockito.when(context.getAttribute(HttpServer.ADMINS_ACL)).thenReturn(acls); - Assert.assertFalse(HttpServer.isInstrumentationAccessAllowed(context, request, response)); + assertFalse(HttpServer.isInstrumentationAccessAllowed(context, request, response)); } @Test diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServerLifecycle.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServerLifecycle.java index 63c99387c00d..420604824628 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServerLifecycle.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServerLifecycle.java @@ -17,38 +17,33 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Ignore("Hangs on occasion; see HBASE-14430") -@Category({ MiscTests.class, SmallTests.class }) +@Disabled("Hangs on occasion; see HBASE-14430") +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestHttpServerLifecycle extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHttpServerLifecycle.class); - /** * Check that a server is alive by probing the {@link HttpServer#isAlive()} method and the text of * its toString() description * @param server server */ private void assertAlive(HttpServer server) { - assertTrue("Server is not alive", server.isAlive()); + assertTrue(server.isAlive(), "Server is not alive"); assertToStringContains(server, HttpServer.STATE_DESCRIPTION_ALIVE); } private void assertNotLive(HttpServer server) { - assertTrue("Server should not be live", !server.isAlive()); + assertTrue(!server.isAlive(), "Server should not be live"); assertToStringContains(server, HttpServer.STATE_DESCRIPTION_NOT_LIVE); } @@ -88,8 +83,8 @@ public void testStartedServerIsAlive() throws Throwable { */ private void assertToStringContains(HttpServer server, String text) { String description = server.toString(); - assertTrue("Did not find \"" + text + "\" in \"" + description + "\"", - description.contains(text)); + assertTrue(description.contains(text), + "Did not find \"" + text + "\" in \"" + description + "\""); } /** @@ -132,6 +127,6 @@ public void testWepAppContextAfterServerStop() throws Throwable { assertAlive(server); assertEquals(value, server.getAttribute(key)); stop(server); - assertNull("Server context should have cleared", server.getAttribute(key)); + assertNull(server.getAttribute(key), "Server context should have cleared"); } } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServerWebapps.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServerWebapps.java index 85aec180c11c..a836c67726c9 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServerWebapps.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServerWebapps.java @@ -17,28 +17,23 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import java.io.FileNotFoundException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Test webapp loading */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestHttpServerWebapps extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHttpServerWebapps.class); - private static final Logger log = LoggerFactory.getLogger(TestHttpServerWebapps.class); /** diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java index 459865509630..91a3321bdfce 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.net.HttpURLConnection; @@ -29,21 +29,20 @@ import org.apache.directory.server.core.annotations.CreatePartition; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.http.resource.JerseyResource; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Test class for admin ACLs with LDAP authentication on the HttpServer. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) @CreateLdapServer( transports = { @CreateTransport(protocol = "LDAP", address = LdapConstants.LDAP_SERVER_ADDR), }) @CreateDS(name = "TestLdapAdminACL", allowAnonAccess = true, @@ -57,16 +56,13 @@ "objectClass: inetOrgPerson", "uid: jdoe", "userPassword: secure123" }) public class TestLdapAdminACL extends LdapServerTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLdapAdminACL.class); private static final Logger LOG = LoggerFactory.getLogger(TestLdapAdminACL.class); private static final String ADMIN_CREDENTIALS = "bjones:p@ssw0rd"; private static final String NON_ADMIN_CREDENTIALS = "jdoe:secure123"; private static final String WRONG_CREDENTIALS = "bjones:password"; - @BeforeClass + @BeforeAll public static void setupServer() throws Exception { Configuration conf = new Configuration(); setLdapConfigurationWithACLs(conf); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java index bff4dc9d9591..c4936513fb36 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.net.HttpURLConnection; @@ -27,17 +27,16 @@ import org.apache.directory.server.core.annotations.ContextEntry; import org.apache.directory.server.core.annotations.CreateDS; import org.apache.directory.server.core.annotations.CreatePartition; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test class for LDAP authentication on the HttpServer. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) @CreateLdapServer( transports = { @CreateTransport(protocol = "LDAP", address = LdapConstants.LDAP_SERVER_ADDR), }) @CreateDS(name = "TestLdapHttpServer", allowAnonAccess = true, @@ -48,10 +47,6 @@ "objectClass: inetOrgPerson", "uid: bjones", "userPassword: p@ssw0rd" }) public class TestLdapHttpServer extends LdapServerTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLdapHttpServer.class); - private static final String BJONES_CREDENTIALS = "bjones:p@ssw0rd"; private static final String WRONG_CREDENTIALS = "bjones:password"; diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestPathFilter.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestPathFilter.java index 75d798cb0778..bcb147dacafe 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestPathFilter.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestPathFilter.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Set; @@ -30,21 +30,17 @@ import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.net.NetUtils; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestPathFilter extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestPathFilter.class); private static final Logger LOG = LoggerFactory.getLogger(HttpServer.class); private static final Set RECORDS = new TreeSet<>(); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestProfileOutputServlet.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestProfileOutputServlet.java index 442fc0e37fe7..cfa4c8877205 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestProfileOutputServlet.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestProfileOutputServlet.java @@ -17,22 +17,17 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.Arrays; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ SmallTests.class }) +@Tag(SmallTests.TAG) public class TestProfileOutputServlet { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProfileOutputServlet.class); @Test public void testSanitization() { @@ -43,12 +38,8 @@ public void testSanitization() { } List bad = Arrays.asList("function(){console.log(\"oops\")}", "uhoh"); for (String input : bad) { - try { - ProfileOutputServlet.sanitize(input); - fail("Expected sanitization of \"" + input + "\" to fail"); - } catch (RuntimeException e) { - // Pass - } + assertThrows(RuntimeException.class, () -> ProfileOutputServlet.sanitize(input), + "Expected sanitization of \"" + input + "\" to fail"); } } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestProxyUserSpnegoHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestProxyUserSpnegoHttpServer.java index e6842bbb02ec..90f498acd05b 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestProxyUserSpnegoHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestProxyUserSpnegoHttpServer.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.net.HttpURLConnection; @@ -31,7 +31,6 @@ import javax.security.auth.Subject; import javax.security.auth.kerberos.KerberosTicket; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.http.TestHttpServer.EchoServlet; import org.apache.hadoop.hbase.http.resource.JerseyResource; @@ -62,11 +61,10 @@ import org.ietf.jgss.GSSManager; import org.ietf.jgss.GSSName; import org.ietf.jgss.Oid; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,11 +73,9 @@ * HttpComponents to verify that the doas= mechanicsm works, and that the proxyuser settings are * observed. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestProxyUserSpnegoHttpServer extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProxyUserSpnegoHttpServer.class); private static final Logger LOG = LoggerFactory.getLogger(TestProxyUserSpnegoHttpServer.class); private static final String KDC_SERVER_HOST = "localhost"; @@ -97,7 +93,7 @@ public class TestProxyUserSpnegoHttpServer extends HttpServerFunctionalTest { private static File privilegedKeytab; private static File privileged2Keytab; - @BeforeClass + @BeforeAll public static void setupServer() throws Exception { Configuration conf = new Configuration(); HBaseCommonTestingUtility htu = new HBaseCommonTestingUtility(conf); @@ -137,7 +133,7 @@ public static void setupServer() throws Exception { LOG.info("HTTP server started: " + baseUrl); } - @AfterClass + @AfterAll public static void stopServer() throws Exception { try { if (null != server) { diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSSLHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSSLHttpServer.java index 4654a27d87ae..6c27c21b762d 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSSLHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSSLHttpServer.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.ByteArrayOutputStream; import java.io.File; @@ -29,7 +29,6 @@ import javax.net.ssl.HttpsURLConnection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; @@ -38,11 +37,10 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.ssl.SSLFactory; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,13 +48,10 @@ * This testcase issues SSL certificates configures the HttpServer to serve HTTPS using the created * certficates and calls an echo servlet using the corresponding HTTPS URL. */ -@Category({ MiscTests.class, MediumTests.class }) +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) public class TestSSLHttpServer extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSSLHttpServer.class); - private static final Logger LOG = LoggerFactory.getLogger(TestSSLHttpServer.class); private static Configuration serverConf; private static HttpServer server; @@ -66,7 +61,7 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest { private static SSLFactory clientSslFactory; private static HBaseCommonTestingUtility HTU; - @BeforeClass + @BeforeAll public static void setup() throws Exception { HTU = new HBaseCommonTestingUtility(); @@ -106,7 +101,7 @@ public static void setup() throws Exception { LOG.info("HTTP server started: " + baseUrl); } - @AfterClass + @AfterAll public static void cleanup() throws Exception { server.stop(); FileUtil.fullyDelete(new File(HTU.getDataTestDir().toString())); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSecurityHeadersFilter.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSecurityHeadersFilter.java index 006025e0a978..ec50921d8cda 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSecurityHeadersFilter.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSecurityHeadersFilter.java @@ -28,25 +28,19 @@ import java.net.HttpURLConnection; import java.net.URL; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.hamcrest.core.Is; import org.hamcrest.core.IsEqual; -import org.junit.After; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ HttpServerFunctionalTest.class, MediumTests.class }) +@Tag(MediumTests.TAG) public class TestSecurityHeadersFilter { private static URL baseUrl; private HttpServer http; - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSecurityHeadersFilter.class); - - @After + @AfterEach public void tearDown() throws Exception { http.stop(); } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestServletFilter.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestServletFilter.java index 1b95e0dfe236..711f213eeee7 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestServletFilter.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestServletFilter.java @@ -17,9 +17,10 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Random; @@ -32,24 +33,19 @@ import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestServletFilter extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestServletFilter.class); private static final Logger LOG = LoggerFactory.getLogger(HttpServer.class); private static volatile String uri = null; @@ -94,12 +90,12 @@ public void initFilter(FilterContainer container, Configuration conf) { private static void assertExceptionContains(String string, Throwable t) { String msg = t.getMessage(); - Assert.assertTrue("Expected to find '" + string + "' but got unexpected exception:" - + StringUtils.stringifyException(t), msg.contains(string)); + assertTrue(msg.contains(string), "Expected to find '" + string + + "' but got unexpected exception:" + StringUtils.stringifyException(t)); } @Test - @Ignore + @Disabled // From stack // Its a 'foreign' test, one that came in from hadoop when we copy/pasted http // It's second class. Could comment it out if only failing test (as per @nkeywal – sort of) @@ -169,12 +165,8 @@ public void testServletFilterWhenInitThrowsException() throws Exception { // start an http server with ErrorFilter conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY, ErrorFilter.Initializer.class.getName()); HttpServer http = createTestServer(conf); - try { - http.start(); - fail("expecting exception"); - } catch (IOException e) { - assertExceptionContains("Problem starting http server", e); - } + IOException e = assertThrows(IOException.class, () -> http.start()); + assertExceptionContains("Problem starting http server", e); } /** @@ -187,11 +179,7 @@ public void testContextSpecificServletFilterWhenInitThrowsException() throws Exc HttpServer http = createTestServer(conf); HttpServer.defineFilter(http.webAppContext, "ErrorFilter", ErrorFilter.class.getName(), null, null); - try { - http.start(); - fail("expecting exception"); - } catch (IOException e) { - assertExceptionContains("Unable to initialize WebAppContext", e); - } + IOException e = assertThrows(IOException.class, () -> http.start()); + assertExceptionContains("Unable to initialize WebAppContext", e); } } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSpnegoHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSpnegoHttpServer.java index 68f8d88ddff2..6d7c82157807 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSpnegoHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSpnegoHttpServer.java @@ -17,9 +17,10 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.File; import java.io.IOException; @@ -31,7 +32,6 @@ import javax.security.auth.Subject; import javax.security.auth.kerberos.KerberosTicket; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.http.TestHttpServer.EchoServlet; import org.apache.hadoop.hbase.http.resource.JerseyResource; @@ -61,11 +61,10 @@ import org.ietf.jgss.GSSManager; import org.ietf.jgss.GSSName; import org.ietf.jgss.Oid; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,11 +72,9 @@ * Test class for SPNEGO authentication on the HttpServer. Uses Kerby's MiniKDC and Apache * HttpComponents to verify that a simple Servlet is reachable via SPNEGO and unreachable w/o. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestSpnegoHttpServer extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSpnegoHttpServer.class); private static final Logger LOG = LoggerFactory.getLogger(TestSpnegoHttpServer.class); private static final String KDC_SERVER_HOST = "localhost"; @@ -89,7 +86,7 @@ public class TestSpnegoHttpServer extends HttpServerFunctionalTest { private static File infoServerKeytab; private static File clientKeytab; - @BeforeClass + @BeforeAll public static void setupServer() throws Exception { Configuration conf = new Configuration(); HBaseCommonTestingUtility htu = new HBaseCommonTestingUtility(conf); @@ -121,7 +118,7 @@ public static void setupServer() throws Exception { LOG.info("HTTP server started: " + baseUrl); } - @AfterClass + @AfterAll public static void stopServer() throws Exception { try { if (null != server) { @@ -221,7 +218,7 @@ public HttpResponse run() throws Exception { assertEquals("a:b", EntityUtils.toString(resp.getEntity()).trim()); } - @Test(expected = IllegalArgumentException.class) + @Test public void testMissingConfigurationThrowsException() throws Exception { Configuration conf = new Configuration(); conf.setInt(HttpServer.HTTP_MAX_THREADS, TestHttpServer.MAX_THREADS); @@ -229,9 +226,12 @@ public void testMissingConfigurationThrowsException() throws Exception { conf.set("hbase.security.authentication", "kerberos"); // Intentionally skip keytab and principal - HttpServer customServer = createTestServerWithSecurity(conf); - customServer.addUnprivilegedServlet("echo", "/echo", EchoServlet.class); - customServer.addJerseyResourcePackage(JerseyResource.class.getPackage().getName(), "/jersey/*"); - customServer.start(); + assertThrows(IllegalArgumentException.class, () -> { + HttpServer customServer = createTestServerWithSecurity(conf); + customServer.addUnprivilegedServlet("echo", "/echo", EchoServlet.class); + customServer.addJerseyResourcePackage(JerseyResource.class.getPackage().getName(), + "/jersey/*"); + customServer.start(); + }); } } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java index fe202906e9c5..7bed30d721ec 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.http.conf; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.StringReader; import java.io.StringWriter; @@ -29,12 +29,10 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -47,13 +45,10 @@ * Basic test case that the ConfServlet can write configuration to its output in XML and JSON * format. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestConfServlet { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestConfServlet.class); - private static final String TEST_KEY = "testconfservlet.key"; private static final String TEST_VAL = "testval"; @@ -125,12 +120,8 @@ public void testMask() { @Test public void testBadFormat() throws Exception { StringWriter sw = new StringWriter(); - try { - ConfServlet.writeResponse(getTestConf(), sw, "not a format"); - fail("writeResponse with bad format didn't throw!"); - } catch (ConfServlet.BadFormatException bfe) { - // expected - } + assertThrows(ConfServlet.BadFormatException.class, + () -> ConfServlet.writeResponse(getTestConf(), sw, "not a format")); assertEquals("", sw.toString()); } } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/jmx/TestJMXJsonServlet.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/jmx/TestJMXJsonServlet.java index 7577d5cfd40a..a0092c859169 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/jmx/TestJMXJsonServlet.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/jmx/TestJMXJsonServlet.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.http.jmx; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.HttpURLConnection; import java.net.URL; @@ -27,31 +27,26 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.http.HttpServer; import org.apache.hadoop.hbase.http.HttpServerFunctionalTest; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestJMXJsonServlet extends HttpServerFunctionalTest { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestJMXJsonServlet.class); - private static final Logger LOG = LoggerFactory.getLogger(TestJMXJsonServlet.class); private static HttpServer server; private static URL baseUrl; - @BeforeClass + @BeforeAll public static void setup() throws Exception { // Eclipse doesn't pick this up correctly from the plugin // configuration in the pom. @@ -61,7 +56,7 @@ public static void setup() throws Exception { baseUrl = getServerURL(server); } - @AfterClass + @AfterAll public static void cleanup() throws Exception { server.stop(); } @@ -69,13 +64,13 @@ public static void cleanup() throws Exception { public static void assertReFind(String re, String value) { Pattern p = Pattern.compile(re); Matcher m = p.matcher(value); - assertTrue("'" + p + "' does not match " + value, m.find()); + assertTrue(m.find(), "'" + p + "' does not match " + value); } public static void assertNotFind(String re, String value) { Pattern p = Pattern.compile(re); Matcher m = p.matcher(value); - assertFalse("'" + p + "' should not match " + value, m.find()); + assertFalse(m.find(), "'" + p + "' should not match " + value); } @Test diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestAuthenticationFilterInitializer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestAuthenticationFilterInitializer.java index 68c48f282737..6ffef18b25eb 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestAuthenticationFilterInitializer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestAuthenticationFilterInitializer.java @@ -17,29 +17,26 @@ */ package org.apache.hadoop.hbase.http.lib; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.ArgumentMatchers.any; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.http.FilterContainer; import org.apache.hadoop.hbase.http.HttpServer; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestAuthenticationFilterInitializer { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAuthenticationFilterInitializer.class); @Test public void testConfiguration() throws Exception { diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestStaticUserWebFilter.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestStaticUserWebFilter.java index 9ad5ab9e9d9f..22faab984f8e 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestStaticUserWebFilter.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/lib/TestStaticUserWebFilter.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.http.lib; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -30,23 +30,18 @@ import javax.servlet.http.HttpServletRequestWrapper; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.http.ServerConfigurationKeys; import org.apache.hadoop.hbase.http.lib.StaticUserWebFilter.StaticUserFilter; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestStaticUserWebFilter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStaticUserWebFilter.class); - private FilterConfig mockConfig(String username) { FilterConfig mock = mock(FilterConfig.class); doReturn(username).when(mock).getInitParameter(ServerConfigurationKeys.HBASE_HTTP_STATIC_USER); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java index 23beacfba74b..ac48b89f2787 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.hbase.http.log; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.File; import java.io.IOException; @@ -30,6 +31,8 @@ import java.net.URI; import java.security.PrivilegedExceptionAction; import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.net.ssl.SSLException; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.FileUtils; @@ -38,12 +41,12 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.http.HttpConfig; import org.apache.hadoop.hbase.http.HttpServer; import org.apache.hadoop.hbase.http.log.LogLevel.CLI; import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; +import org.apache.hadoop.hbase.logging.Log4jUtils; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -54,20 +57,17 @@ import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test LogLevel. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestLogLevel { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLogLevel.class); private static String keystoresDir; private static String sslConfDir; @@ -89,8 +89,34 @@ public class TestLogLevel { private static String HTTP_PRINCIPAL = "HTTP/" + LOCALHOST; private static HBaseCommonTestingUtility HTU; private static File keyTabFile; + private static final Pattern EFFECTIVE_LEVEL = Pattern.compile("Effective level:\\s*(\\S+)"); - @BeforeClass + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + + @FunctionalInterface + private interface ThrowingConsumer { + void accept(String url) throws Exception; + } + + private enum Protocol { + C_HTTP_S_HTTP(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP), + C_HTTP_S_HTTPS(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS), + C_HTTPS_S_HTTP(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP), + C_HTTPS_S_HTTPS(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS); + + final String client; + final String server; + + Protocol(String client, String server) { + this.client = client; + this.server = server; + } + } + + @BeforeAll public static void setUp() throws Exception { serverConf = new Configuration(); serverConf.setStrings(LogLevel.READONLY_LOGGERS_CONF_KEY, protectedPrefix); @@ -170,7 +196,7 @@ private static Configuration getSslConfig(Configuration conf) { return sslConf; } - @AfterClass + @AfterAll public static void tearDown() { if (kdc != null) { kdc.stop(); @@ -261,36 +287,44 @@ private HttpServer createServer(String protocol, boolean isSpnego) throws Except return server; } - private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol, - final boolean isSpnego) throws Exception { - testDynamicLogLevel(bindProtocol, connectProtocol, isSpnego, logName, - org.apache.logging.log4j.Level.DEBUG.toString()); + private void testGetLogLevel(Protocol protocol, boolean isSpnego, String loggerName, + String expectedLevel) throws Exception { + withLogLevelServer(protocol, isSpnego, (authority) -> { + final String level = getLevel(protocol.client, authority, loggerName); + assertEquals(expectedLevel, level, "Log level not equal to expected: "); + }); } - private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol, - final boolean isSpnego, final String newLevel) throws Exception { - testDynamicLogLevel(bindProtocol, connectProtocol, isSpnego, logName, newLevel); + private void testSetLogLevel(Protocol protocol, boolean isSpnego, String loggerName, + String newLevel) throws Exception { + String oldLevel = Log4jUtils.getEffectiveLevel(loggerName); + assertNotEquals(newLevel, oldLevel, "New level is same as old level: "); + + try { + withLogLevelServer(protocol, isSpnego, (authority) -> { + setLevel(protocol.client, authority, loggerName, newLevel); + }); + } finally { + // restore log level + Log4jUtils.setLogLevel(loggerName, oldLevel); + } } /** - * Run both client and server using the given protocol. - * @param bindProtocol specify either http or https for server - * @param connectProtocol specify either http or https for client - * @param isSpnego true if SPNEGO is enabled - * @throws Exception if client can't accesss server. + * Starts a LogLevel server and executes a client action against it. + * @param protocol protocol configuration for client and server + * @param isSpnego whether SPNEGO authentication is enabled + * @param consumer client action executed with the server authority (host:port) + * @throws Exception if server setup or client execution fails */ - private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol, - final boolean isSpnego, final String loggerName, final String newLevel) throws Exception { - if (!LogLevel.isValidProtocol(bindProtocol)) { - throw new Exception("Invalid server protocol " + bindProtocol); + private void withLogLevelServer(Protocol protocol, final boolean isSpnego, + ThrowingConsumer consumer) throws Exception { + if (!LogLevel.isValidProtocol(protocol.server)) { + throw new Exception("Invalid server protocol " + protocol.server); } - if (!LogLevel.isValidProtocol(connectProtocol)) { - throw new Exception("Invalid client protocol " + connectProtocol); + if (!LogLevel.isValidProtocol(protocol.client)) { + throw new Exception("Invalid client protocol " + protocol.client); } - org.apache.logging.log4j.Logger log = org.apache.logging.log4j.LogManager.getLogger(loggerName); - org.apache.logging.log4j.Level oldLevel = log.getLevel(); - assertNotEquals("Get default Log Level which shouldn't be ERROR.", - org.apache.logging.log4j.Level.ERROR, oldLevel); // configs needed for SPNEGO at server side if (isSpnego) { @@ -305,28 +339,21 @@ private void testDynamicLogLevel(final String bindProtocol, final String connect UserGroupInformation.setConfiguration(serverConf); } - final HttpServer server = createServer(bindProtocol, isSpnego); - // get server port + final HttpServer server = createServer(protocol.server, isSpnego); final String authority = NetUtils.getHostPortString(server.getConnectorAddress(0)); - String keytabFilePath = keyTabFile.getAbsolutePath(); UserGroupInformation clientUGI = UserGroupInformation.loginUserFromKeytabAndReturnUGI(clientPrincipal, keytabFilePath); try { clientUGI.doAs((PrivilegedExceptionAction) () -> { - // client command line - getLevel(connectProtocol, authority, loggerName); - setLevel(connectProtocol, authority, loggerName, newLevel); + consumer.accept(authority); return null; }); } finally { clientUGI.logoutUserFromKeytab(); server.stop(); } - - // restore log level - org.apache.logging.log4j.core.config.Configurator.setLevel(log.getName(), oldLevel); } /** @@ -335,10 +362,12 @@ private void testDynamicLogLevel(final String bindProtocol, final String connect * @param authority daemon's web UI address * @throws Exception if unable to connect */ - private void getLevel(String protocol, String authority, String logName) throws Exception { + private String getLevel(String protocol, String authority, String logName) throws Exception { String[] getLevelArgs = { "-getlevel", authority, logName, "-protocol", protocol }; CLI cli = new CLI(protocol.equalsIgnoreCase("https") ? sslConf : clientConf); - cli.run(getLevelArgs); + cli.parseArguments(getLevelArgs); + final String response = cli.fetchGetLevelResponse(); + return extractEffectiveLevel(response); } /** @@ -351,19 +380,33 @@ private void setLevel(String protocol, String authority, String logName, String throws Exception { String[] setLevelArgs = { "-setlevel", authority, logName, newLevel, "-protocol", protocol }; CLI cli = new CLI(protocol.equalsIgnoreCase("https") ? sslConf : clientConf); - cli.run(setLevelArgs); + cli.parseArguments(setLevelArgs); + final String response = cli.fetchSetLevelResponse(); + final String responseLevel = extractEffectiveLevel(response); + final String currentLevel = Log4jUtils.getEffectiveLevel(logName); + assertEquals(newLevel, currentLevel, "new level not equal to expected: "); + assertSame(newLevel, responseLevel, "new level not equal to response level: "); + } - org.apache.logging.log4j.Logger logger = org.apache.logging.log4j.LogManager.getLogger(logName); + /** + * Extract effective log level from server response. + * @param response server body response string + * @return the effective log level + */ + private String extractEffectiveLevel(String response) { + Matcher m = EFFECTIVE_LEVEL.matcher(response); + if (m.find()) { + return org.apache.logging.log4j.Level.toLevel(m.group(1)).name(); + } - assertEquals("new level not equal to expected: ", newLevel.toUpperCase(), - logger.getLevel().toString()); + fail("Cannot find effective log level from response: " + response); + return null; } @Test public void testSettingProtectedLogLevel() throws Exception { try { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, protectedLogName, - "DEBUG"); + testSetLogLevel(Protocol.C_HTTP_S_HTTP, true, protectedLogName, "DEBUG"); fail("Expected IO exception due to protected logger"); } catch (IOException e) { assertTrue(e.getMessage().contains("" + HttpServletResponse.SC_PRECONDITION_FAILED)); @@ -372,13 +415,26 @@ public void testSettingProtectedLogLevel() throws Exception { } } + @Test + public void testGetDebugLogLevel() throws Exception { + Log4jUtils.setLogLevel(logName, "DEBUG"); + testGetLogLevel(Protocol.C_HTTP_S_HTTP, true, logName, "DEBUG"); + } + + @Test + public void testGetInfoLogLevel() throws Exception { + Log4jUtils.setLogLevel(logName, "INFO"); + testGetLogLevel(Protocol.C_HTTP_S_HTTP, true, logName, "INFO"); + } + /** * Test setting log level to "Info". * @throws Exception if client can't set log level to INFO. */ @Test - public void testInfoLogLevel() throws Exception { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, "INFO"); + public void testSetInfoLogLevel() throws Exception { + Log4jUtils.setLogLevel(logName, "DEBUG"); + testSetLogLevel(Protocol.C_HTTP_S_HTTP, true, logName, "INFO"); } /** @@ -386,8 +442,9 @@ public void testInfoLogLevel() throws Exception { * @throws Exception if client can't set log level to ERROR. */ @Test - public void testErrorLogLevel() throws Exception { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, "ERROR"); + public void testSetErrorLogLevel() throws Exception { + Log4jUtils.setLogLevel(logName, "DEBUG"); + testSetLogLevel(Protocol.C_HTTP_S_HTTP, true, logName, "ERROR"); } /** @@ -397,10 +454,11 @@ public void testErrorLogLevel() throws Exception { */ @Test public void testLogLevelByHttp() throws Exception { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false); + Log4jUtils.setLogLevel(logName, "DEBUG"); + testGetLogLevel(Protocol.C_HTTP_S_HTTP, false, logName, "DEBUG"); try { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS, false); - fail("An HTTPS Client should not have succeeded in connecting to a " + "HTTP server"); + testGetLogLevel(Protocol.C_HTTPS_S_HTTP, false, logName, "DEBUG"); + fail("An HTTPS Client should not have succeeded in connecting to a HTTP server"); } catch (SSLException e) { exceptionShouldContains("Unrecognized SSL message", e); } @@ -413,10 +471,11 @@ public void testLogLevelByHttp() throws Exception { */ @Test public void testLogLevelByHttpWithSpnego() throws Exception { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true); + Log4jUtils.setLogLevel(logName, "DEBUG"); + testGetLogLevel(Protocol.C_HTTP_S_HTTP, true, logName, "DEBUG"); try { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS, true); - fail("An HTTPS Client should not have succeeded in connecting to a " + "HTTP server"); + testGetLogLevel(Protocol.C_HTTPS_S_HTTP, true, logName, "DEBUG"); + fail("An HTTPS Client should not have succeeded in connecting to a HTTP server"); } catch (SSLException e) { exceptionShouldContains("Unrecognized SSL message", e); } @@ -429,10 +488,11 @@ public void testLogLevelByHttpWithSpnego() throws Exception { */ @Test public void testLogLevelByHttps() throws Exception { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS, false); + Log4jUtils.setLogLevel(logName, "DEBUG"); + testGetLogLevel(Protocol.C_HTTPS_S_HTTPS, false, logName, "DEBUG"); try { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP, false); - fail("An HTTP Client should not have succeeded in connecting to a " + "HTTPS server"); + testGetLogLevel(Protocol.C_HTTP_S_HTTPS, false, logName, "DEBUG"); + fail("An HTTP Client should not have succeeded in connecting to a HTTPS server"); } catch (SocketException e) { exceptionShouldContains("Unexpected end of file from server", e); } @@ -445,15 +505,56 @@ public void testLogLevelByHttps() throws Exception { */ @Test public void testLogLevelByHttpsWithSpnego() throws Exception { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS, true); + Log4jUtils.setLogLevel(logName, "DEBUG"); + testGetLogLevel(Protocol.C_HTTPS_S_HTTPS, true, logName, "DEBUG"); try { - testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP, true); - fail("An HTTP Client should not have succeeded in connecting to a " + "HTTPS server"); + testGetLogLevel(Protocol.C_HTTP_S_HTTPS, true, logName, "DEBUG"); + fail("An HTTP Client should not have succeeded in connecting to a HTTPS server"); } catch (SocketException e) { exceptionShouldContains("Unexpected end of file from server", e); } } + /** + * Test getting log level in readonly mode. + * @throws Exception if a client can't get log level. + */ + @Test + public void testGetLogLevelAllowedInReadonlyMode() throws Exception { + withMasterUIReadonly(() -> { + Log4jUtils.setLogLevel(logName, "DEBUG"); + testGetLogLevel(Protocol.C_HTTP_S_HTTP, true, logName, "DEBUG"); + }); + } + + /** + * Test setting log level in readonly mode. + * @throws Exception if a client can set log level. + */ + @Test + public void testSetLogLevelDisallowedInReadonlyMode() throws Exception { + withMasterUIReadonly(() -> { + Log4jUtils.setLogLevel(logName, "DEBUG"); + try { + testSetLogLevel(Protocol.C_HTTP_S_HTTP, true, logName, "INFO"); + fail("Setting log level should be disallowed in readonly mode."); + } catch (IOException e) { + exceptionShouldContains("Modification of HBase via the UI is disallowed in configuration.", + e); + } + }); + } + + private void withMasterUIReadonly(ThrowingRunnable runnable) throws Exception { + boolean prev = serverConf.getBoolean(LogLevel.MASTER_UI_READONLY_CONF_KEY, false); + serverConf.setBoolean(LogLevel.MASTER_UI_READONLY_CONF_KEY, true); + try { + runnable.run(); + } finally { + serverConf.setBoolean(LogLevel.MASTER_UI_READONLY_CONF_KEY, prev); + } + } + /** * Assert that a throwable or one of its causes should contain the substr in its message. Ideally * we should use {@link GenericTestUtils#assertExceptionContains(String, Throwable)} util method diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/prometheus/TestPrometheusServlet.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/prometheus/TestPrometheusServlet.java index 276a2b9ad296..2790b0d631d2 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/prometheus/TestPrometheusServlet.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/prometheus/TestPrometheusServlet.java @@ -18,11 +18,11 @@ package org.apache.hadoop.hbase.http.prometheus; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.metrics2.MetricsSystem; @@ -30,21 +30,16 @@ import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test prometheus Sink. */ -@Category({ SmallTests.class, MiscTests.class }) +@Tag(SmallTests.TAG) +@Tag(MiscTests.TAG) public class TestPrometheusServlet { - @ClassRule - public static final HBaseClassTestRule CLASS_TEST_RULE = - HBaseClassTestRule.forClass(TestPrometheusServlet.class); - @Test public void testPublish() throws IOException { // GIVEN @@ -66,8 +61,8 @@ public void testPublish() throws IOException { // THEN String writtenMetrics = stream.toString(UTF_8.name()); System.out.println(writtenMetrics); - Assert.assertTrue("The expected metric line is missing from prometheus metrics output", - writtenMetrics.contains("test_metrics_num_bucket_create_fails{context=\"dfs\"")); + assertTrue(writtenMetrics.contains("test_metrics_num_bucket_create_fails{context=\"dfs\""), + "The expected metric line is missing from prometheus metrics output"); metrics.stop(); metrics.shutdown(); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/util/TestJSONBean.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/util/TestJSONBean.java index ef36d19c0468..66132f395109 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/util/TestJSONBean.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/util/TestJSONBean.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; @@ -33,12 +33,10 @@ import javax.management.MBeanInfo; import javax.management.MBeanServer; import javax.management.ObjectName; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.reflect.TypeToken; import org.apache.hbase.thirdparty.com.google.gson.Gson; @@ -46,11 +44,9 @@ /** * Test {@link JSONBean}. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestJSONBean { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestJSONBean.class); private MBeanServer getMockMBeanServer() throws Exception { MBeanServer mbeanServer = mock(MBeanServer.class); diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index e75219d1ed1d..b4821a74d4b3 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -186,8 +186,18 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test @@ -285,7 +295,7 @@ maven-failsafe-plugin false - always + false 1800 diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestAcidGuarantees.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestAcidGuarantees.java index 6516494278ee..603037825c5d 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestAcidGuarantees.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestAcidGuarantees.java @@ -28,8 +28,8 @@ import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; @@ -45,7 +45,7 @@ * -DnumGetters=2 -DnumScanners=2 -DnumUniqueRows=5 * */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestAcidGuarantees extends IntegrationTestBase { private static final int SERVER_COUNT = 1; // number of slaves for the smallest cluster diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java index 4326c9852e35..3db3060a56ff 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.charset.Charset; @@ -51,11 +52,10 @@ import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.ToolRunner; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +70,7 @@ * @see HBASE-7912 * @see HBASE-14123 */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestBackupRestore extends IntegrationTestBase { private static final String CLASS_NAME = IntegrationTestBackupRestore.class.getSimpleName(); protected static final Logger LOG = LoggerFactory.getLogger(IntegrationTestBackupRestore.class); @@ -84,7 +84,7 @@ public class IntegrationTestBackupRestore extends IntegrationTestBase { protected static final int DEFAULT_REGIONSERVER_COUNT = 5; protected static final int DEFAULT_NUMBER_OF_TABLES = 1; protected static final int DEFAULT_NUM_ITERATIONS = 10; - protected static final int DEFAULT_ROWS_IN_ITERATION = 500000; + protected static final int DEFAULT_ROWS_IN_ITERATION = 10000; protected static final String SLEEP_TIME_KEY = "sleeptime"; // short default interval because tests don't run very long. protected static final long SLEEP_TIME_DEFAULT = 50000L; @@ -101,8 +101,40 @@ public class IntegrationTestBackupRestore extends IntegrationTestBase { private static String BACKUP_ROOT_DIR = "backupIT"; + /* + * This class is used to run the backup and restore thread(s). Throwing an exception in this + * thread will not cause the test to fail, so the purpose of this class is to both kick off the + * backup and restore and record any exceptions that occur so they can be thrown in the main + * thread. + */ + protected class BackupAndRestoreThread implements Runnable { + private final TableName table; + private Throwable throwable; + + public BackupAndRestoreThread(TableName table) { + this.table = table; + this.throwable = null; + } + + public Throwable getThrowable() { + return this.throwable; + } + + @Override + public void run() { + try { + runTestSingle(this.table); + } catch (Throwable t) { + LOG.error( + "An error occurred in thread {} when performing a backup and restore with table {}: ", + Thread.currentThread().getName(), this.table.getNameAsString(), t); + this.throwable = t; + } + } + } + @Override - @Before + @BeforeEach public void setUp() throws Exception { util = new IntegrationTestingUtility(); Configuration conf = util.getConfiguration(); @@ -118,7 +150,7 @@ public void setUp() throws Exception { LOG.info("Cluster initialized and ready"); } - @After + @AfterEach public void tearDown() throws IOException { LOG.info("Cleaning up after test."); if (util.isDistributedCluster()) { @@ -174,28 +206,35 @@ public void testBackupRestore() throws Exception { runTestMulti(); } - private void runTestMulti() throws IOException { + private void runTestMulti() throws Exception { LOG.info("IT backup & restore started"); Thread[] workers = new Thread[numTables]; + BackupAndRestoreThread[] backupAndRestoreThreads = new BackupAndRestoreThread[numTables]; for (int i = 0; i < numTables; i++) { final TableName table = tableNames[i]; - Runnable r = new Runnable() { - @Override - public void run() { - try { - runTestSingle(table); - } catch (IOException e) { - LOG.error("Failed", e); - Assert.fail(e.getMessage()); - } - } - }; - workers[i] = new Thread(r); + BackupAndRestoreThread backupAndRestoreThread = new BackupAndRestoreThread(table); + backupAndRestoreThreads[i] = backupAndRestoreThread; + workers[i] = new Thread(backupAndRestoreThread); workers[i].start(); } - // Wait all workers to finish - for (Thread t : workers) { - Uninterruptibles.joinUninterruptibly(t); + // Wait for all workers to finish and check for errors + Throwable error = null; + Throwable threadThrowable; + for (int i = 0; i < numTables; i++) { + Uninterruptibles.joinUninterruptibly(workers[i]); + threadThrowable = backupAndRestoreThreads[i].getThrowable(); + if (threadThrowable == null) { + continue; + } + if (error == null) { + error = threadThrowable; + } else { + error.addSuppressed(threadThrowable); + } + } + // Throw any found errors after all threads have completed + if (error != null) { + throw new AssertionError("An error occurred in a backup and restore thread", error); } LOG.info("IT backup & restore finished"); } @@ -229,8 +268,7 @@ private void loadData(TableName table, int numRows) throws IOException { } private String backup(BackupRequest request, BackupAdmin client) throws IOException { - String backupId = client.backupTables(request); - return backupId; + return client.backupTables(request); } private void restore(RestoreRequest request, BackupAdmin client) throws IOException { @@ -292,7 +330,7 @@ private void runTestSingle(TableName table) throws IOException { restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tablesRestoreIncMultiple, null, true), client); Table hTable = conn.getTable(table); - Assert.assertEquals(util.countRows(hTable), rowsInIteration * numIterations); + assertEquals(util.countRows(hTable), rowsInIteration * numIterations); hTable.close(); LOG.info("{} loop {} finished.", Thread.currentThread().getName(), (count - 1)); } @@ -300,13 +338,12 @@ private void runTestSingle(TableName table) throws IOException { private void restoreVerifyTable(Connection conn, BackupAdmin client, TableName table, String backupId, long expectedRows) throws IOException { - TableName[] tablesRestoreIncMultiple = new TableName[] { table }; restore( createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tablesRestoreIncMultiple, null, true), client); Table hTable = conn.getTable(table); - Assert.assertEquals(expectedRows, util.countRows(hTable)); + assertEquals(expectedRows, util.countRows(hTable)); hTable.close(); } @@ -351,7 +388,7 @@ public RestoreRequest createRestoreRequest(String backupRootDir, String backupId TableName[] fromTables, TableName[] toTables, boolean isOverwrite) { RestoreRequest.Builder builder = new RestoreRequest.Builder(); return builder.withBackupRootDir(backupRootDir).withBackupId(backupId).withCheck(check) - .withFromTables(fromTables).withToTables(toTables).withOvewrite(isOverwrite).build(); + .withFromTables(fromTables).withToTables(toTables).withOverwrite(isOverwrite).build(); } @Override diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java index b94170d79952..fb52d426854d 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java @@ -27,8 +27,8 @@ import org.apache.hadoop.hbase.chaos.factories.MonkeyFactory; import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.util.AbstractHBaseTool; -import org.junit.After; -import org.junit.Before; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -159,13 +159,13 @@ protected int doWork() throws Exception { return result; } - @Before + @BeforeEach public void setUp() throws Exception { setUpCluster(); setUpMonkey(); } - @After + @AfterEach public void cleanUp() throws Exception { cleanUpMonkey(); cleanUpCluster(); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.java index dacbdb8af88f..bdd7a681fe60 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -191,8 +193,8 @@ protected void verifyNamespaces() throws IOException { // iterating concurrent map for (String nsName : namespaceMap.keySet()) { try { - Assert.assertTrue("Namespace: " + nsName + " in namespaceMap does not exist", - admin.getNamespaceDescriptor(nsName) != null); + assertTrue(admin.getNamespaceDescriptor(nsName) != null, + "Namespace: " + nsName + " in namespaceMap does not exist"); } catch (NamespaceNotFoundException nsnfe) { Assert .fail("Namespace: " + nsName + " in namespaceMap does not exist: " + nsnfe.getMessage()); @@ -206,12 +208,12 @@ protected void verifyTables() throws IOException { Admin admin = connection.getAdmin(); // iterating concurrent map for (TableName tableName : enabledTables.keySet()) { - Assert.assertTrue("Table: " + tableName + " in enabledTables is not enabled", - admin.isTableEnabled(tableName)); + assertTrue(admin.isTableEnabled(tableName), + "Table: " + tableName + " in enabledTables is not enabled"); } for (TableName tableName : disabledTables.keySet()) { - Assert.assertTrue("Table: " + tableName + " in disabledTables is not disabled", - admin.isTableDisabled(tableName)); + assertTrue(admin.isTableDisabled(tableName), + "Table: " + tableName + " in disabledTables is not disabled"); } for (TableName tableName : deletedTables.keySet()) { Assert.assertFalse("Table: " + tableName + " in deletedTables is not deleted", @@ -287,7 +289,7 @@ void perform() throws IOException { LOG.info("Creating namespace:" + nsd); admin.createNamespace(nsd); NamespaceDescriptor freshNamespaceDesc = admin.getNamespaceDescriptor(nsd.getName()); - Assert.assertTrue("Namespace: " + nsd + " was not created", freshNamespaceDesc != null); + assertTrue(freshNamespaceDesc != null, "Namespace: " + nsd + " was not created"); namespaceMap.put(nsd.getName(), freshNamespaceDesc); LOG.info("Created namespace:" + freshNamespaceDesc); } catch (Exception e) { @@ -329,10 +331,10 @@ void perform() throws IOException { modifiedNsd.setConfiguration(nsTestConfigKey, nsValueNew); admin.modifyNamespace(modifiedNsd); NamespaceDescriptor freshNamespaceDesc = admin.getNamespaceDescriptor(namespaceName); - Assert.assertTrue("Namespace: " + selected + " was not modified", - freshNamespaceDesc.getConfigurationValue(nsTestConfigKey).equals(nsValueNew)); - Assert.assertTrue("Namespace: " + namespaceName + " does not exist", - admin.getNamespaceDescriptor(namespaceName) != null); + assertTrue(freshNamespaceDesc.getConfigurationValue(nsTestConfigKey).equals(nsValueNew), + "Namespace: " + selected + " was not modified"); + assertTrue(admin.getNamespaceDescriptor(namespaceName) != null, + "Namespace: " + namespaceName + " does not exist"); namespaceMap.put(namespaceName, freshNamespaceDesc); LOG.info("Modified namespace :" + freshNamespaceDesc); } catch (Exception e) { @@ -360,7 +362,7 @@ void perform() throws IOException { try { if (admin.getNamespaceDescriptor(namespaceName) != null) { // the namespace still exists. - Assert.assertTrue("Namespace: " + selected + " was not deleted", false); + assertTrue(false, "Namespace: " + selected + " was not deleted"); } else { LOG.info("Deleted namespace :" + selected); } @@ -411,10 +413,10 @@ void perform() throws IOException { byte[] endKey = Bytes.toBytes("row-" + Integer.MAX_VALUE); LOG.info("Creating table:" + td); admin.createTable(td, startKey, endKey, numRegions); - Assert.assertTrue("Table: " + td + " was not created", admin.tableExists(tableName)); + assertTrue(admin.tableExists(tableName), "Table: " + td + " was not created"); TableDescriptor freshTableDesc = admin.getDescriptor(tableName); - Assert.assertTrue("After create, Table: " + tableName + " in not enabled", - admin.isTableEnabled(tableName)); + assertTrue(admin.isTableEnabled(tableName), + "After create, Table: " + tableName + " in not enabled"); enabledTables.put(tableName, freshTableDesc); LOG.info("Created table:" + freshTableDesc); } catch (Exception e) { @@ -449,11 +451,10 @@ void perform() throws IOException { TableName tableName = selected.getTableName(); LOG.info("Disabling table :" + selected); admin.disableTable(tableName); - Assert.assertTrue("Table: " + selected + " was not disabled", - admin.isTableDisabled(tableName)); + assertTrue(admin.isTableDisabled(tableName), "Table: " + selected + " was not disabled"); TableDescriptor freshTableDesc = admin.getDescriptor(tableName); - Assert.assertTrue("After disable, Table: " + tableName + " is not disabled", - admin.isTableDisabled(tableName)); + assertTrue(admin.isTableDisabled(tableName), + "After disable, Table: " + tableName + " is not disabled"); disabledTables.put(tableName, freshTableDesc); LOG.info("Disabled table :" + freshTableDesc); } catch (Exception e) { @@ -497,11 +498,10 @@ void perform() throws IOException { TableName tableName = selected.getTableName(); LOG.info("Enabling table :" + selected); admin.enableTable(tableName); - Assert.assertTrue("Table: " + selected + " was not enabled", - admin.isTableEnabled(tableName)); + assertTrue(admin.isTableEnabled(tableName), "Table: " + selected + " was not enabled"); TableDescriptor freshTableDesc = admin.getDescriptor(tableName); - Assert.assertTrue("After enable, Table: " + tableName + " in not enabled", - admin.isTableEnabled(tableName)); + assertTrue(admin.isTableEnabled(tableName), + "After enable, Table: " + tableName + " in not enabled"); enabledTables.put(tableName, freshTableDesc); LOG.info("Enabled table :" + freshTableDesc); } catch (Exception e) { @@ -594,10 +594,10 @@ void perform() throws IOException { admin.addColumnFamily(tableName, cfd); // assertion TableDescriptor freshTableDesc = admin.getDescriptor(tableName); - Assert.assertTrue("Column family: " + cfd + " was not added", - freshTableDesc.hasColumnFamily(cfd.getName())); - Assert.assertTrue("After add column family, Table: " + tableName + " is not disabled", - admin.isTableDisabled(tableName)); + assertTrue(freshTableDesc.hasColumnFamily(cfd.getName()), + "Column family: " + cfd + " was not added"); + assertTrue(admin.isTableDisabled(tableName), + "After add column family, Table: " + tableName + " is not disabled"); disabledTables.put(tableName, freshTableDesc); LOG.info("Added column family: " + cfd + " to table: " + tableName); } catch (Exception e) { @@ -648,9 +648,8 @@ void perform() throws IOException { freshColumnDesc.getMaxVersions(), versions); Assert.assertEquals("Column family: " + freshColumnDesc + " was not altered", freshColumnDesc.getMinVersions(), versions); - Assert.assertTrue( - "After alter versions of column family, Table: " + tableName + " is not disabled", - admin.isTableDisabled(tableName)); + assertTrue(admin.isTableDisabled(tableName), + "After alter versions of column family, Table: " + tableName + " is not disabled"); disabledTables.put(tableName, freshTableDesc); LOG.info("Altered versions of column family: " + columnDesc + " to: " + versions + " in table: " + tableName); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java index c4935d36f010..43765511fbf7 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngest.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -30,9 +33,8 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +45,7 @@ * A base class for tests that do something with the cluster while running {@link LoadTestTool} to * write and verify some data. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestIngest extends IntegrationTestBase { public static final char HIPHEN = '-'; private static final int SERVER_COUNT = 1; // number of slaves for the smallest cluster @@ -95,7 +97,7 @@ protected int getMinServerCount() { protected void initTable() throws IOException { int ret = loadTool.run(getArgsForLoadTestToolInitTable()); - Assert.assertEquals("Failed to initialize LoadTestTool", 0, ret); + assertEquals(0, ret, "Failed to initialize LoadTestTool"); } @Override @@ -175,7 +177,7 @@ protected void runIngestTest(long defaultRunTime, long keysPerServerPerIter, int if (0 != ret) { String errorMsg = "Load failed with error code " + ret; LOG.error(errorMsg); - Assert.fail(errorMsg); + fail(errorMsg); } ret = loadTool.run(getArgsForLoadTestTool("-update", String.format("60:%d:1", writeThreads), @@ -183,7 +185,7 @@ protected void runIngestTest(long defaultRunTime, long keysPerServerPerIter, int if (0 != ret) { String errorMsg = "Update failed with error code " + ret; LOG.error(errorMsg); - Assert.fail(errorMsg); + fail(errorMsg); } ret = loadTool.run( @@ -197,7 +199,7 @@ protected void runIngestTest(long defaultRunTime, long keysPerServerPerIter, int if (0 != ret) { LOG.error("Rerun of Verification failed with error code " + ret); } - Assert.fail(errorMsg); + fail(errorMsg); } startKey += numKeys; } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestStripeCompactions.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestStripeCompactions.java index a9b7bd60b4e6..0e63f4ab119f 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestStripeCompactions.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestStripeCompactions.java @@ -25,13 +25,13 @@ import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.HFileTestUtil; import org.apache.hadoop.util.ToolRunner; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; /** * A test class that does the same things as IntegrationTestIngest but with stripe compactions. Can * be used with ChaosMonkey in the same manner. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestIngestStripeCompactions extends IntegrationTestIngest { @Override protected void initTable() throws IOException { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java index 6b7566496ce9..bfa66381d9ff 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithACL.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.util.LoadTestTool; import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL; import org.apache.hadoop.util.ToolRunner; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; @@ -39,7 +39,7 @@ * WRITE permissions are not read back and cells with READ permissions are read back. Every * operation happens in the user's specific context */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestIngestWithACL extends IntegrationTestIngest { private static final char COLON = ':'; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java index e0dcb0c48582..4b3442830040 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java @@ -29,12 +29,12 @@ import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.EncryptionTest; import org.apache.hadoop.util.ToolRunner; -import org.junit.Before; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestIngestWithEncryption extends IntegrationTestIngest { private final static Logger LOG = LoggerFactory.getLogger(IntegrationTestIngestWithEncryption.class); @@ -63,7 +63,7 @@ public void setUpCluster() throws Exception { initialized = true; } - @Before + @BeforeEach @Override public void setUp() throws Exception { // Initialize the cluster. This invokes LoadTestTool -init_only, which diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithMOB.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithMOB.java index 75846c3c14cc..6ad4a7388ff4 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithMOB.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithMOB.java @@ -33,15 +33,15 @@ import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithMOB; import org.apache.hadoop.hbase.util.LoadTestTool; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; /** * Integration Test for MOB ingest. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestIngestWithMOB extends IntegrationTestIngest { private static final char COLON = ':'; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithTags.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithTags.java index 870ab1b47415..1f9b7000c6a9 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithTags.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithTags.java @@ -24,9 +24,9 @@ import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithTags; import org.apache.hadoop.hbase.util.LoadTestTool; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestIngestWithTags extends IntegrationTestIngest { private static final char COLON = ':'; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithVisibilityLabels.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithVisibilityLabels.java index 1993d7950f6a..23b635cf4cce 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithVisibilityLabels.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithVisibilityLabels.java @@ -28,9 +28,9 @@ import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.LoadTestTool; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestIngestWithVisibilityLabels extends IntegrationTestIngest { private static final char COMMA = ','; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java index 8cf644b12605..318ee2322994 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.security.InvalidParameterException; import java.util.Map; import java.util.Set; @@ -39,11 +42,10 @@ import org.apache.hadoop.hbase.util.MultiThreadedWriter; import org.apache.hadoop.hbase.util.RegionSplitter; import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +53,7 @@ * Integration test that verifies lazy CF loading during scans by doing repeated scans with this * feature while multiple threads are continuously writing values; and verifying the result. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestLazyCfLoading { private static final TableName TABLE_NAME = TableName.valueOf(IntegrationTestLazyCfLoading.class.getSimpleName()); @@ -169,7 +171,7 @@ public Filter getScanFilter() { } } - @Before + @BeforeEach public void setUp() throws Exception { LOG.info("Initializing cluster with " + NUM_SERVERS + " servers"); util.initializeCluster(NUM_SERVERS); @@ -208,7 +210,7 @@ private void deleteTable() throws Exception { } } - @After + @AfterEach public void tearDown() throws Exception { deleteTable(); LOG.info("Restoring the cluster"); @@ -264,18 +266,17 @@ public void testReadersAndWriters() throws Exception { // Verify and count the results. while ((result = results.next()) != null) { boolean isOk = writer.verifyResultAgainstDataGenerator(result, true, true); - Assert.assertTrue("Failed to verify [" + Bytes.toString(result.getRow()) + "]", isOk); + assertTrue(isOk, "Failed to verify [" + Bytes.toString(result.getRow()) + "]"); ++resultCount; } long timeTaken = EnvironmentEdgeManager.currentTime() - startTs; // Verify the result count. long onesGennedAfterScan = dataGen.getExpectedNumberOfKeys(); - Assert.assertTrue( - "Read " + resultCount + " keys when at most " + onesGennedAfterScan + " were generated ", - onesGennedAfterScan >= resultCount); + assertTrue(onesGennedAfterScan >= resultCount, + "Read " + resultCount + " keys when at most " + onesGennedAfterScan + " were generated "); if (isWriterDone) { - Assert.assertTrue("Read " + resultCount + " keys; the writer is done and " - + onesGennedAfterScan + " keys were generated", onesGennedAfterScan == resultCount); + assertTrue(onesGennedAfterScan == resultCount, "Read " + resultCount + + " keys; the writer is done and " + onesGennedAfterScan + " keys were generated"); } else if (onesGennedBeforeScan * 0.9 > resultCount) { LOG.warn("Read way too few keys (" + resultCount + "/" + onesGennedBeforeScan + ") - there might be a problem, or the writer might just be slow"); @@ -286,8 +287,8 @@ public void testReadersAndWriters() throws Exception { now = EnvironmentEdgeManager.currentTime(); } } - Assert.assertEquals("There are write failures", 0, writer.getNumWriteFailures()); - Assert.assertTrue("Writer is not done", isWriterDone); + assertEquals(0, writer.getNumWriteFailures(), "There are write failures"); + assertTrue(isWriterDone, "Writer is not done"); // Assert.fail("Boom!"); connection.close(); } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestManyRegions.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestManyRegions.java index 722a1e8ea594..23dd72f6d813 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestManyRegions.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestManyRegions.java @@ -18,19 +18,15 @@ package org.apache.hadoop.hbase; import java.io.IOException; -import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.RegionSplitter; import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +35,7 @@ * verify it completes within a reasonable amount of time. * @see HBASE-7220 */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestManyRegions { private static final String CLASS_NAME = IntegrationTestManyRegions.class.getSimpleName(); @@ -64,14 +60,10 @@ public class IntegrationTestManyRegions { protected static final int DEFAULT_TIMEOUT_MINUTES = 5; protected static final int TIMEOUT_MINUTES = UTIL.getConfiguration().getInt(TIMEOUT_MINUTES_KEY, DEFAULT_TIMEOUT_MINUTES); - // This timeout is suitable since there is only single testcase in this test. - @ClassRule - public static final TestRule timeout = Timeout.builder() - .withTimeout(TIMEOUT_MINUTES, TimeUnit.MINUTES).withLookingForStuckThread(true).build(); private Admin admin; - @Before + @BeforeEach public void setUp() throws Exception { LOG.info(String.format("Initializing cluster with %d region servers.", REGION_SERVER_COUNT)); UTIL.initializeCluster(REGION_SERVER_COUNT); @@ -87,7 +79,7 @@ public void setUp() throws Exception { LOG.info("Cluster ready"); } - @After + @AfterEach public void tearDown() throws IOException { LOG.info("Cleaning up after test."); if (admin.tableExists(TABLE_NAME)) { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java index c709c0db89d4..374287c2546d 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java @@ -26,10 +26,10 @@ import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hbase.zookeeper.ZNodePaths; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +39,7 @@ * requests on the created table - the other replicas of the meta would be used to get the location * of the region of the created table. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestMetaReplicas { private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestMetaReplicas.class); /** @@ -47,7 +47,7 @@ public class IntegrationTestMetaReplicas { */ private static IntegrationTestingUtility util; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { // Set up the integration test util if (util == null) { @@ -71,7 +71,7 @@ public static void setUp() throws Exception { waitUntilZnodeAvailable(2); } - @AfterClass + @AfterAll public static void teardown() throws Exception { // Clean everything up. util.restoreCluster(); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java index 208f11807b06..6111f7d04ee2 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -42,10 +42,10 @@ import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.ToolRunner; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,8 +68,7 @@ * */ @SuppressWarnings("deprecation") - -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestMobCompaction extends IntegrationTestBase { protected static final Logger LOG = LoggerFactory.getLogger(IntegrationTestMobCompaction.class); @@ -101,7 +100,7 @@ public class IntegrationTestMobCompaction extends IntegrationTestBase { private static volatile boolean run = true; @Override - @Before + @BeforeEach public void setUp() throws Exception { util = getTestingUtil(getConf()); conf = util.getConfiguration(); @@ -128,7 +127,7 @@ private void createTestTable() throws IOException { table = util.createTable(hdt, null); } - @After + @AfterEach public void tearDown() throws IOException { LOG.info("Cleaning up after test."); if (util.isDistributedCluster()) { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java index 9d5664db36d4..25622cc03318 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hbase; import static java.lang.String.format; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.codahale.metrics.Histogram; import java.util.ArrayDeque; @@ -44,7 +44,7 @@ import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.ToolRunner; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +58,7 @@ * IntegrationTestBase is incompatible with the JUnit runner. Hence no @Test annotations either. See * {@code -help} for full list of options. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestRegionReplicaPerf extends IntegrationTestBase { private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestRegionReplicaPerf.class); @@ -164,12 +164,12 @@ public void setUp() throws Exception { // sanity check cluster // TODO: this should reach out to master and verify online state instead - assertEquals("Master must be configured with StochasticLoadBalancer", - "org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer", - conf.get("hbase.master.loadbalancer.class")); + assertEquals("org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer", + conf.get("hbase.master.loadbalancer.class"), + "Master must be configured with StochasticLoadBalancer"); // TODO: this should reach out to master and verify online state instead - assertTrue("hbase.regionserver.storefile.refresh.period must be greater than zero.", - conf.getLong("hbase.regionserver.storefile.refresh.period", 0) > 0); + assertTrue(conf.getLong("hbase.regionserver.storefile.refresh.period", 0) > 0, + "hbase.regionserver.storefile.refresh.period must be greater than zero."); // enable client-side settings conf.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, true); @@ -241,7 +241,7 @@ private static double calcMean(String desc, Stat stat, List result for (TimingResult tr : results) { for (PerformanceEvaluation.RunResult r : tr.results) { - assertNotNull("One of the run results is missing detailed run data.", r.hist); + assertNotNull(r.hist, "One of the run results is missing detailed run data."); sum += stat.apply(r.hist); count += 1; LOG.debug(desc + "{" + YammerHistogramUtils.getHistogramReport(r.hist) + "}"); @@ -269,9 +269,9 @@ public void test() throws Exception { new PerfEvalCallable(util.getAdmin(), writeOpts).call(); // one last sanity check, then send in the clowns! - assertEquals("Table must be created with DisabledRegionSplitPolicy. Broken test.", - DisabledRegionSplitPolicy.class.getName(), - util.getAdmin().getTableDescriptor(tableName).getRegionSplitPolicyClassName()); + assertEquals(DisabledRegionSplitPolicy.class.getName(), + util.getAdmin().getTableDescriptor(tableName).getRegionSplitPolicyClassName(), + "Table must be created with DisabledRegionSplitPolicy. Broken test."); startMonkey(); // collect a baseline without region replicas. @@ -313,16 +313,14 @@ public void test() throws Exception { .add("withReplicasStdevMean", withReplicasStdevMean) .add("withReplicas99.99Mean", withReplicas9999Mean).toString()); - assertTrue( + assertTrue(withReplicasStdevMean <= withoutReplicasStdevMean, "Running with region replicas under chaos should have less request variance than without. " + "withReplicas.stdev.mean: " + withReplicasStdevMean + "ms " - + "withoutReplicas.stdev.mean: " + withoutReplicasStdevMean + "ms.", - withReplicasStdevMean <= withoutReplicasStdevMean); - assertTrue( + + "withoutReplicas.stdev.mean: " + withoutReplicasStdevMean + "ms."); + assertTrue(withReplicas9999Mean <= withoutReplicas9999Mean, "Running with region replicas under chaos should improve 99.99pct latency. " + "withReplicas.99.99.mean: " + withReplicas9999Mean + "ms " - + "withoutReplicas.99.99.mean: " + withoutReplicas9999Mean + "ms.", - withReplicas9999Mean <= withoutReplicas9999Mean); + + "withoutReplicas.99.99.mean: " + withoutReplicas9999Mean + "ms."); } public static void main(String[] args) throws Exception { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java index 5817d071f02c..9746136852f7 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java @@ -28,7 +28,6 @@ import java.util.regex.Pattern; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.junit.rules.ExternalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,10 +41,9 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.util.RegexSet; /** - * A {@link org.junit.Rule} that manages a simple http server. The caller registers request handlers - * to URI path regexp. + * A simple http server for testing. The caller registers request handlers to URI path regexp. */ -public class MockHttpApiRule extends ExternalResource { +public class MockHttpApiRule { private static final Logger LOG = LoggerFactory.getLogger(MockHttpApiRule.class); private MockHandler handler; @@ -92,8 +90,7 @@ public URI getURI() { return server.getURI(); } - @Override - protected void before() throws Exception { + public void start() throws Exception { handler = new MockHandler(); server = new Server(); final ServerConnector http = new ServerConnector(server); @@ -105,8 +102,7 @@ protected void before() throws Exception { server.start(); } - @Override - protected void after() { + public void close() { try { server.stop(); } catch (Exception e) { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/TestIntegrationTestBase.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/TestIntegrationTestBase.java index 469f64937200..12e26528aee9 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/TestIntegrationTestBase.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/TestIntegrationTestBase.java @@ -17,23 +17,18 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.chaos.factories.MonkeyConstants; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestIntegrationTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestIntegrationTestBase.class); - @Test public void testMonkeyPropertiesParsing() { final Configuration conf = new Configuration(false); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/TestRESTApiClusterManager.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/TestRESTApiClusterManager.java index 71a2530cb690..00b65e60c859 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/TestRESTApiClusterManager.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/TestRESTApiClusterManager.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.HashMap; @@ -26,48 +26,46 @@ import org.apache.hadoop.hbase.ClusterManager.ServiceType; import org.apache.hadoop.hbase.RESTApiClusterManager.Service; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestRESTApiClusterManager { - @ClassRule - public static final HBaseClassTestRule testRule = - HBaseClassTestRule.forClass(TestRESTApiClusterManager.class); - - @ClassRule public static MockHttpApiRule mockHttpApi = new MockHttpApiRule(); - @Rule - public final TestName testName = new TestName(); - private static HBaseCommonTestingUtility testingUtility; private ClusterManager clusterManager; - @BeforeClass - public static void beforeClass() { + @BeforeAll + public static void beforeClass() throws Exception { + mockHttpApi.start(); testingUtility = new HBaseCommonTestingUtility(); configureClusterManager(testingUtility.getConfiguration()); } - @Before - public void before() { + @AfterAll + public static void afterClass() throws Exception { + mockHttpApi.close(); + } + + @BeforeEach + public void before(TestInfo testInfo) { mockHttpApi.clearRegistrations(); final Configuration methodConf = new Configuration(testingUtility.getConfiguration()); - methodConf.set("hbase.it.clustermanager.restapi.clustername", testName.getMethodName()); + methodConf.set("hbase.it.clustermanager.restapi.clustername", + testInfo.getTestMethod().get().getName()); clusterManager = new RESTApiClusterManager(); clusterManager.setConf(methodConf); } @Test - public void isRunningPositive() throws IOException { - final String clusterName = testName.getMethodName(); + public void isRunningPositive(TestInfo testInfo) throws IOException { + final String clusterName = testInfo.getTestMethod().get().getName(); final String hostName = "somehost"; final String serviceName = "hbase"; final String hostId = "some-id"; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/TestShellExecEndpointCoprocessor.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/TestShellExecEndpointCoprocessor.java index 2831d5c52270..84c6afdae993 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/TestShellExecEndpointCoprocessor.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/TestShellExecEndpointCoprocessor.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.IOException; @@ -29,35 +29,53 @@ import java.nio.file.Paths; import java.util.Optional; import java.util.function.Consumer; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.AsyncAdmin; import org.apache.hadoop.hbase.client.AsyncConnection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecRequest; import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecResponse; import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecService; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test for the {@link ShellExecEndpointCoprocessor}. */ -@Category(MediumTests.class) +@Tag(MediumTests.TAG) public class TestShellExecEndpointCoprocessor { - @ClassRule - public static final HBaseClassTestRule testRule = - HBaseClassTestRule.forClass(TestShellExecEndpointCoprocessor.class); + private static HBaseTestingUtility testingUtility; + private AsyncConnection conn; - @ClassRule - public static final MiniClusterRule miniClusterRule = - MiniClusterRule.newBuilder().setConfiguration(createConfiguration()).build(); + @BeforeAll + public static void setUp() throws Exception { + testingUtility = new HBaseTestingUtility(); + testingUtility.getConfiguration().set("hbase.coprocessor.master.classes", + ShellExecEndpointCoprocessor.class.getName()); + testingUtility.startMiniCluster(); + } + + @AfterAll + public static void tearDown() throws Exception { + testingUtility.shutdownMiniCluster(); + } + + @BeforeEach + public void setUpConnection() throws Exception { + conn = ConnectionFactory.createAsyncConnection(testingUtility.getConfiguration()).get(); + } - @Rule - public final ConnectionRule connectionRule = - ConnectionRule.createAsyncConnectionRule(miniClusterRule::createAsyncConnection); + @AfterEach + public void tearDownConnection() throws IOException { + if (conn != null) { + conn.close(); + } + } @Test public void testShellExecUnspecified() { @@ -71,7 +89,6 @@ public void testShellExecForeground() { } private void testShellExecForeground(final Consumer consumer) { - final AsyncConnection conn = connectionRule.getAsyncConnection(); final AsyncAdmin admin = conn.getAdmin(); final String command = "echo -n \"hello world\""; @@ -87,10 +104,9 @@ admin. coprocessorService(ShellExecSer @Test public void testShellExecBackground() throws IOException { - final AsyncConnection conn = connectionRule.getAsyncConnection(); final AsyncAdmin admin = conn.getAdmin(); - final File testDataDir = ensureTestDataDirExists(miniClusterRule.getTestingUtility()); + final File testDataDir = ensureTestDataDirExists(testingUtility); final File testFile = new File(testDataDir, "shell_exec_background.txt"); assertTrue(testFile.createNewFile()); assertEquals(0, testFile.length()); @@ -102,9 +118,9 @@ public void testShellExecBackground() throws IOException { admin. coprocessorService(ShellExecService::newStub, (stub, controller, callback) -> stub.shellExec(controller, req, callback)).join(); - assertFalse("the response from a background task should have no exit code", resp.hasExitCode()); - assertFalse("the response from a background task should have no stdout", resp.hasStdout()); - assertFalse("the response from a background task should have no stderr", resp.hasStderr()); + assertFalse(resp.hasExitCode(), "the response from a background task should have no exit code"); + assertFalse(resp.hasStdout(), "the response from a background task should have no stdout"); + assertFalse(resp.hasStderr(), "the response from a background task should have no stderr"); Waiter.waitFor(conn.getConfiguration(), 5_000, () -> testFile.length() > 0); final String content = @@ -121,10 +137,4 @@ private static File ensureTestDataDirExists(final HBaseTestingUtility testingUti assertTrue(testDataDirFile.exists()); return testDataDirFile; } - - private static Configuration createConfiguration() { - final Configuration conf = HBaseConfiguration.create(); - conf.set("hbase.coprocessor.master.classes", ShellExecEndpointCoprocessor.class.getName()); - return conf; - } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java index 30babcd4d413..f56d23df3c65 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java @@ -122,6 +122,8 @@ public void perform() throws Exception { suspendRs(server); } catch (Shell.ExitCodeException e) { LOG.warn("Problem suspending but presume successful; code={}", e.getExitCode(), e); + } catch (Exception e) { + LOG.warn("Problem suspending but presume successful", e); } suspendedServers.add(server); break; @@ -131,6 +133,8 @@ public void perform() throws Exception { resumeRs(server); } catch (Shell.ExitCodeException e) { LOG.info("Problem resuming, will retry; code={}", e.getExitCode(), e); + } catch (Exception e) { + LOG.warn("Problem resulting, will retry", e); } resumedServers.add(server); break; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/TestChangeSplitPolicyAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/TestChangeSplitPolicyAction.java index 35aee4c52053..420aa1070edb 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/TestChangeSplitPolicyAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/TestChangeSplitPolicyAction.java @@ -17,42 +17,36 @@ */ package org.apache.hadoop.hbase.chaos.actions; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.IntegrationTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; -@Category({ MediumTests.class }) +@Tag(MediumTests.TAG) public class TestChangeSplitPolicyAction { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestChangeSplitPolicyAction.class); - private final static IntegrationTestingUtility TEST_UTIL = new IntegrationTestingUtility(); private final TableName tableName = TableName.valueOf("ChangeSplitPolicyAction"); - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(2); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } - @Before + @BeforeEach public void setUp() throws Exception { Admin admin = TEST_UTIL.getAdmin(); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java index fb8ab209c3a1..756f0d3846a6 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java @@ -86,7 +86,6 @@ private static ExecutorService buildMonkeyThreadPool(final int size) { return Executors.newFixedThreadPool(size, new ThreadFactoryBuilder().setDaemon(false) .setNameFormat("ChaosMonkey-%d").setUncaughtExceptionHandler((t, e) -> { LOG.error("Uncaught exception in thread {}", t.getName(), e); - throw new RuntimeException(e); }).build()); } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java index b021de2d73e7..beb33601b92a 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java @@ -20,9 +20,9 @@ import static org.apache.hadoop.hbase.ipc.RpcClient.SPECIFIC_WRITE_THREAD; import static org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.SERVICE; import static org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.newBlockingStub; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.net.InetSocketAddress; @@ -41,9 +41,9 @@ import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface; import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.Threads; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +53,7 @@ import org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos.EchoResponseProto; import org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface; -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestRpcClient { private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestRpcClient.class); @@ -330,7 +330,7 @@ public Void call() throws Exception { } @Test - @Ignore // TODO: test fails with async client + @Disabled // TODO: test fails with async client public void testRpcWithChaosMonkeyWithAsyncClient() throws Throwable { for (int i = 0; i < numIterations; i++) { TimeoutThread.runWithTimeout(new Callable() { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java index e84eb3940403..17740ce91b65 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.mapreduce; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.DataInput; import java.io.DataOutput; @@ -78,8 +78,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,7 +100,7 @@ * hbase.IntegrationTestBulkLoad.replicaCount How many region replicas to configure for the table * under test. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestBulkLoad extends IntegrationTestBase { private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestBulkLoad.class); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java index abb1ae297815..4267fdfefae3 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestFileBasedSFTBulkLoad.java @@ -23,8 +23,8 @@ import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +42,7 @@ * hbase.IntegrationTestBulkLoad.replicaCount How many region replicas to configure for the table * under test. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestFileBasedSFTBulkLoad extends IntegrationTestBulkLoad { private static final Logger LOG = diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java index 9a5773325246..42111485e782 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hbase.mapreduce; import static java.lang.String.format; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.IOException; @@ -50,12 +50,11 @@ import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +64,7 @@ /** * Validate ImportTsv + LoadIncrementalHFiles on a distributed cluster. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestImportTsv extends Configured implements Tool { private static final String NAME = IntegrationTestImportTsv.class.getSimpleName(); @@ -77,9 +76,6 @@ public class IntegrationTestImportTsv extends Configured implements Tool { + "row3\t1\tc1\tc2\n" + "row4\t1\tc1\tc2\n" + "row5\t1\tc1\tc2\n" + "row6\t1\tc1\tc2\n" + "row7\t1\tc1\tc2\n" + "row8\t1\tc1\tc2\n" + "row9\t1\tc1\tc2\n" + "row10\t1\tc1\tc2\n"; - @Rule - public TestName name = new TestName(); - protected static final Set simple_expected = new TreeSet(CellComparator.getInstance()) { private static final long serialVersionUID = 1L; @@ -113,7 +109,7 @@ public void setConf(Configuration conf) { LOG.debug("Ignoring setConf call."); } - @BeforeClass + @BeforeAll public static void provisionCluster() throws Exception { if (null == util) { util = new IntegrationTestingUtility(); @@ -125,7 +121,7 @@ public static void provisionCluster() throws Exception { } } - @AfterClass + @AfterAll public static void releaseCluster() throws Exception { util.restoreCluster(); if (!util.isDistributedCluster()) { @@ -141,8 +137,8 @@ protected void doLoadIncrementalHFiles(Path hfiles, TableName tableName) throws String[] args = { hfiles.toString(), tableName.getNameAsString() }; LOG.info(format("Running LoadIncrememntalHFiles with args: %s", Arrays.asList(args))); - assertEquals("Loading HFiles failed.", 0, - ToolRunner.run(new LoadIncrementalHFiles(new Configuration(getConf())), args)); + assertEquals(0, ToolRunner.run(new LoadIncrementalHFiles(new Configuration(getConf())), args), + "Loading HFiles failed."); Table table = null; Scan scan = new Scan(); @@ -156,14 +152,14 @@ protected void doLoadIncrementalHFiles(Path hfiles, TableName tableName) throws while (resultsIt.hasNext() && expectedIt.hasNext()) { Result r = resultsIt.next(); for (Cell actual : r.rawCells()) { - assertTrue("Ran out of expected values prematurely!", expectedIt.hasNext()); + assertTrue(expectedIt.hasNext(), "Ran out of expected values prematurely!"); KeyValue expected = expectedIt.next(); - assertEquals("Scan produced surprising result", 0, - CellComparator.getInstance().compare(expected, actual)); + assertEquals(0, CellComparator.getInstance().compare(expected, actual), + "Scan produced surprising result"); } } - assertFalse("Did not consume all expected values.", expectedIt.hasNext()); - assertFalse("Did not consume all scan results.", resultsIt.hasNext()); + assertFalse(expectedIt.hasNext(), "Did not consume all expected values."); + assertFalse(resultsIt.hasNext(), "Did not consume all scan results."); } finally { if (null != table) table.close(); } @@ -177,12 +173,12 @@ protected static void validateDeletedPartitionsFile(Configuration conf) throws I FileSystem fs = FileSystem.get(conf); Path partitionsFile = new Path(TotalOrderPartitioner.getPartitionFile(conf)); - assertFalse("Failed to clean up partitions file.", fs.exists(partitionsFile)); + assertFalse(fs.exists(partitionsFile), "Failed to clean up partitions file."); } @Test - public void testGenerateAndLoad() throws Exception { - generateAndLoad(TableName.valueOf(name.getMethodName())); + public void testGenerateAndLoad(TestInfo testInfo) throws Exception { + generateAndLoad(TableName.valueOf(testInfo.getTestMethod().get().getName())); } void generateAndLoad(final TableName table) throws Exception { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableMapReduceUtil.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableMapReduceUtil.java index 745b509f457f..c836347a61b9 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableMapReduceUtil.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestTableMapReduceUtil.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.mapreduce; -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -28,32 +28,32 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test that we add tmpjars correctly including the named dependencies. Runs as an integration test * so that classpath is realistic. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestTableMapReduceUtil implements Configurable, Tool { private static IntegrationTestingUtility util; - @BeforeClass + @BeforeAll public static void provisionCluster() throws Exception { if (null == util) { util = new IntegrationTestingUtility(); } } - @Before + @BeforeEach public void skipMiniCluster() { // test probably also works with a local cluster, but // IntegrationTestingUtility doesn't support this concept. - assumeTrue("test requires a distributed cluster.", util.isDistributedCluster()); + assumeTrue(util.isDistributedCluster(), "test requires a distributed cluster."); } /** diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java index d47df570ecdc..5091e99cc6ea 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.mttr; -import static org.junit.Assert.assertEquals; -import static org.junit.Assume.assumeFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeFalse; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Scope; @@ -63,10 +63,10 @@ import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.LoadTestTool; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,7 +102,7 @@ * At the end of the test a log line is output on the INFO level containing the timing data that was * collected. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestMTTR { /** * Constants. @@ -143,7 +143,7 @@ public class IntegrationTestMTTR { */ private static LoadTestTool loadTool; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { // Set up the integration test util if (util == null) { @@ -232,10 +232,10 @@ private static void setupTables() throws IOException { // Setup the table for LoadTestTool int ret = loadTool.run(new String[] { "-tn", loadTableName.getNameAsString(), "-init_only" }); - assertEquals("Failed to initialize LoadTestTool", 0, ret); + assertEquals(0, ret, "Failed to initialize LoadTestTool"); } - @AfterClass + @AfterAll public static void after() throws IOException { // Clean everything up. util.restoreCluster(); @@ -598,7 +598,7 @@ public Boolean call() throws Exception { int ret = loadTool.run(new String[] { "-tn", loadTableName.getNameAsString(), "-write", String.format("%d:%d:%d", colsPerKey, 500, writeThreads), "-num_keys", String.valueOf(numKeys), "-skip_init" }); - assertEquals("Load failed", 0, ret); + assertEquals(0, ret, "Load failed"); } while (!future.isDone()); return true; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index c1854d87c199..063467b94874 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -108,8 +108,8 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -206,7 +206,7 @@ * *

*/ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestBigLinkedList extends IntegrationTestBase { protected static final byte[] NO_KEY = new byte[1]; protected static String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table"; @@ -1532,9 +1532,20 @@ protected void runVerify(String outputDir, int numReducers, long expectedNumNode Verify verify = new Verify(); verify.setConf(getConf()); - int retCode = verify.run(iterationOutput, numReducers); - if (retCode > 0) { - throw new RuntimeException("Verify.run failed with return code: " + retCode); + + int retries = getConf().getInt("hbase.itbll.verify.retries", 1); + + while (true) { + int retCode = verify.run(iterationOutput, numReducers); + if (retCode > 0) { + if (retries-- > 0) { + LOG.warn("Verify.run failed with return code: {}. Will retry", retries); + } else { + throw new RuntimeException("Verify.run failed with return code: " + retCode); + } + } else { + break; + } } if (!verify.verify(expectedNumNodes)) { @@ -1859,7 +1870,7 @@ public void testContinuousIngest() throws IOException, Exception { } int ret = ToolRunner.run(conf, new Loop(), new String[] { "1", "1", "2000000", util.getDataTestDirOnTestFS("IntegrationTestBigLinkedList").toString(), "1" }); - org.junit.Assert.assertEquals(0, ret); + org.junit.jupiter.api.Assertions.assertEquals(0, ret); } private void usage() { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java index 5e3fd82be7b5..625d55e83bd3 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java @@ -65,8 +65,8 @@ import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,7 +90,7 @@ * 20000 /tmp 1 10000 or ./hbase org.apache.hadoop.hbase.IntegrationTestsDriver -r * .*IntegrationTestBigLinkedListWithVisibility.* */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestBigLinkedListWithVisibility extends IntegrationTestBigLinkedList { private static final String CONFIDENTIAL = "confidential"; @@ -631,7 +631,7 @@ public void testContinuousIngest() throws IOException, Exception { new String[] { "1", "1", "20000", util.getDataTestDirOnTestFS("IntegrationTestBigLinkedListWithVisibility").toString(), "1", "10000" }); - org.junit.Assert.assertEquals(0, ret); + org.junit.jupiter.api.Assertions.assertEquals(0, ret); } public static void main(String[] args) throws Exception { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java index 450d866d4b4d..06bd8ad97f49 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedReader; import java.io.FileNotFoundException; @@ -82,8 +82,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,7 +102,7 @@ * hdfs and inspected later. This class can be run as a unit test, as an integration test, or from * the command line Originally taken from Apache Bigtop. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestLoadAndVerify extends IntegrationTestBase { private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestLoadAndVerify.class); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestMonkeys.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestMonkeys.java index ea0ec46f785f..8691d7440b74 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestMonkeys.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestMonkeys.java @@ -24,14 +24,14 @@ import org.apache.hadoop.hbase.chaos.util.Monkeys; import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * This is an integration test for showing a simple usage of how to use {@link Monkeys} to control * {@link ChaosMonkeyRunner}. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestMonkeys extends ChaosMonkeyRunner { private static final int RUN_SECS = 15 * 1000; private static final int WAIT_SECS = 10 * 1000; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestTimeBoundedMultiGetRequestsWithRegionReplicas.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestTimeBoundedMultiGetRequestsWithRegionReplicas.java index 996ddbc9f0bf..af139441050d 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestTimeBoundedMultiGetRequestsWithRegionReplicas.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestTimeBoundedMultiGetRequestsWithRegionReplicas.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.LoadTestTool; import org.apache.hadoop.util.ToolRunner; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; @@ -43,7 +43,7 @@ * conjunction with multiget_batchsize would have different behaviors - the batch of gets goes to * the same region or to multiple regions. */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestTimeBoundedMultiGetRequestsWithRegionReplicas extends IntegrationTestTimeBoundedRequestsWithRegionReplicas { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestTimeBoundedRequestsWithRegionReplicas.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestTimeBoundedRequestsWithRegionReplicas.java index a81fbef6fbfa..58a146ea8d59 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestTimeBoundedRequestsWithRegionReplicas.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestTimeBoundedRequestsWithRegionReplicas.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.test; +import static org.junit.jupiter.api.Assertions.fail; + import java.io.IOException; import java.util.List; import java.util.concurrent.Executors; @@ -48,8 +50,7 @@ import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; -import org.junit.Assert; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +92,7 @@ * -Dhbase.ipc.client.allowsInterrupt=true --monkey serverKilling * */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestTimeBoundedRequestsWithRegionReplicas extends IntegrationTestIngest { private static final Logger LOG = @@ -134,7 +135,7 @@ protected void writeData(int colsPerKey, int recordSize, int writeThreads, long if (0 != ret) { String errorMsg = "Load failed with error code " + ret; LOG.error(errorMsg); - Assert.fail(errorMsg); + fail(errorMsg); } } @@ -222,7 +223,7 @@ public void run() { if (0 != ret) { String errorMsg = "Verification failed with error code " + ret; LOG.error(errorMsg); - Assert.fail(errorMsg); + fail(errorMsg); } } finally { if (result != null) result.cancel(false); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java index 2dfa9350e4c1..c968190b5478 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.security.PrivilegedExceptionAction; @@ -57,7 +57,7 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.ToolRunner; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; import org.apache.hbase.thirdparty.com.google.common.base.Splitter; import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; @@ -74,7 +74,7 @@ * line. Originally taken from Apache Bigtop. Issue user names as comma seperated list. ./hbase * IntegrationTestWithCellVisibilityLoadAndVerify -u usera,userb */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationTestLoadAndVerify { private static final String ERROR_STR = "Two user names are to be specified seperated by a ',' like 'usera,userb'"; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java index 00bfe35983f1..ca84530598cb 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.List; @@ -45,7 +45,7 @@ import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; import org.apache.zookeeper.data.Stat; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,7 +62,7 @@ *

* Example usage: hbase org.apache.hadoop.hbase.test.IntegrationTestZnodeACLs -h */ -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestZKAndFSPermissions extends AbstractHBaseTool { private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestZKAndFSPermissions.class); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/trace/IntegrationTestSendTraceRequests.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/trace/IntegrationTestSendTraceRequests.java index e761ee950dcd..51d5e89b6dda 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/trace/IntegrationTestSendTraceRequests.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/trace/IntegrationTestSendTraceRequests.java @@ -45,14 +45,14 @@ import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; -@Category(IntegrationTests.class) +@Tag(IntegrationTests.TAG) public class IntegrationTestSendTraceRequests extends AbstractHBaseTool { private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestSendTraceRequests.class); public static final String TABLE_ARG = "t"; diff --git a/hbase-logging/pom.xml b/hbase-logging/pom.xml index dd79fb394c97..46ed6801c36b 100644 --- a/hbase-logging/pom.xml +++ b/hbase-logging/pom.xml @@ -42,11 +42,6 @@ org.slf4j slf4j-api - - junit - junit - test - org.slf4j jcl-over-slf4j @@ -100,6 +95,13 @@ net.revelc.code warbucks-maven-plugin + + org.apache.maven.plugins + maven-surefire-plugin + + true + + diff --git a/hbase-mapreduce/pom.xml b/hbase-mapreduce/pom.xml index 40b6cae60da4..b458f7fcedaa 100644 --- a/hbase-mapreduce/pom.xml +++ b/hbase-mapreduce/pom.xml @@ -201,8 +201,28 @@ zookeeper - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility test diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java index 1bbbe513f738..709e9a394533 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java @@ -19,8 +19,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.OptionalLong; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -39,6 +42,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileSplit; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +56,36 @@ public class HFileInputFormat extends FileInputFormat { private static final Logger LOG = LoggerFactory.getLogger(HFileInputFormat.class); + // Configuration key for pluggable location resolver class name + // Used to enable rack-aware processing by providing preferred data locality hints + public static final String CONF_HFILE_LOCATION_RESOLVER_CLASS = + "hfile.backup.input.file.location.resolver.class"; + + /** + * Interface for resolving file locations to influence InputSplit placement for HFile processing. + */ + @InterfaceAudience.Public + public interface HFileLocationResolver { + /** + * Get preferred locations for a group of HFiles to optimize rack-aware processing. + * @param hfiles Collection of HFile paths that will be processed together + * @return Set of preferred host names for processing these HFiles + */ + Set getLocationsForInputFiles(final Collection hfiles); + } + + /** + * Default no-op implementation of HFileLocationResolver. Provides backward compatibility by + * returning no location hints. + */ + public static class NoopHFileLocationResolver implements HFileLocationResolver { + @Override + public Set getLocationsForInputFiles(Collection hfiles) { + // No location hints - lets YARN scheduler decide + return Collections.emptySet(); + } + } + /** * File filter that removes all "hidden" files. This might be something worth removing from a more * general purpose utility; it accounts for the presence of metadata files created in the way @@ -183,4 +217,61 @@ protected boolean isSplitable(JobContext context, Path filename) { // This file isn't splittable. return false; } + + @Override + public List getSplits(JobContext context) throws IOException { + Configuration conf = context.getConfiguration(); + + // Check if location resolver is configured + String resolverClass = conf.get(CONF_HFILE_LOCATION_RESOLVER_CLASS); + if (resolverClass == null) { + LOG.debug("HFile rack-aware processing disabled - no location resolver configured"); + return super.getSplits(context); + } + + LOG.info("HFile rack-aware processing enabled with location resolver: {}", resolverClass); + try { + return createLocationAwareSplits(context, conf); + } catch (Exception e) { + LOG.error("Error creating location-aware splits, falling back to standard splits", e); + return super.getSplits(context); + } + } + + private List createLocationAwareSplits(JobContext context, Configuration conf) + throws IOException { + // Get all HFiles from input paths using existing listStatus logic + List files = listStatus(context); + + // Load configured location resolver + Class resolverClass = + conf.getClass(CONF_HFILE_LOCATION_RESOLVER_CLASS, NoopHFileLocationResolver.class, + HFileLocationResolver.class); + HFileLocationResolver fileLocationResolver = ReflectionUtils.newInstance(resolverClass, conf); + + // Create InputSplits with location hints + List splits = new ArrayList<>(); + + for (FileStatus file : files) { + Path path = file.getPath(); + long length = file.getLen(); + + if (length <= 0) { + LOG.warn("Skipping empty or invalid HFile: {} with length: {}", path, length); + continue; + } + + // Get location hints for this file + Set locations = + fileLocationResolver.getLocationsForInputFiles(Collections.singletonList(path.toString())); + String[] locationArray = locations.toArray(new String[0]); + + splits.add(new FileSplit(path, 0, length, locationArray)); + } + + LOG.info("Created {} location-aware InputSplits from {} HFiles", splits.size(), files.size()); + + return splits; + } + } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 6ab3bdd25048..6e1a28849e13 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -21,8 +21,10 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY; +import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAX_SEQ_ID_KEY; import java.io.IOException; +import java.io.UncheckedIOException; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; import java.net.URLDecoder; @@ -50,6 +52,7 @@ import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -83,6 +86,7 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.OutputFormat; @@ -194,6 +198,11 @@ protected static byte[] combineTableNameSuffix(byte[] tableName, byte[] suffix) "hbase.mapreduce.hfileoutputformat.extendedcell.enabled"; static final boolean EXTENDED_CELL_SERIALIZATION_ENABLED_DEFULT = false; + @InterfaceAudience.Private + public static final String DISK_BASED_SORTING_ENABLED_KEY = + "hbase.mapreduce.hfileoutputformat.disk.based.sorting.enabled"; + private static final boolean DISK_BASED_SORTING_ENABLED_DEFAULT = false; + public static final String REMOTE_CLUSTER_CONF_PREFIX = "hbase.hfileoutputformat.remote.cluster."; public static final String REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY = REMOTE_CLUSTER_CONF_PREFIX + "zookeeper.quorum"; @@ -202,6 +211,13 @@ protected static byte[] combineTableNameSuffix(byte[] tableName, byte[] suffix) public static final String REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY = REMOTE_CLUSTER_CONF_PREFIX + HConstants.ZOOKEEPER_ZNODE_PARENT; + /** + * Set the MAX_SEQ_ID metadata on the resulting HFile. This will ensure the HFiles will be sorted + * properly when read by tools such as the ClientSideRegionScanner. Will have no effect if the + * HFile is bulkloaded, as the sequence ID generated when bulkloading will override this metadata. + */ + public static final String SET_MAX_SEQ_ID_KEY = "hbase.hfileoutputformat.set.max.seq.id"; + public static final String STORAGE_POLICY_PROPERTY = HStore.BLOCK_STORAGE_POLICY_KEY; public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = STORAGE_POLICY_PROPERTY + "."; @@ -263,7 +279,7 @@ static RecordWriter createRecordWrit return new RecordWriter() { // Map of families to writers and how much has been output on the writer. - private final Map writers = new TreeMap<>(Bytes.BYTES_COMPARATOR); + private final Map writers = new TreeMap<>(Bytes.BYTES_COMPARATOR); private final Map previousRows = new TreeMap<>(Bytes.BYTES_COMPARATOR); private final long now = EnvironmentEdgeManager.currentTime(); private byte[] tableNameBytes = writeMultipleTables ? null : Bytes.toBytes(writeTableNames); @@ -293,10 +309,10 @@ public void write(ImmutableBytesWritable row, V cell) throws IOException { } byte[] tableAndFamily = getTableNameSuffixedWithFamily(tableNameBytes, family); - WriterLength wl = this.writers.get(tableAndFamily); + WriterInfo wi = this.writers.get(tableAndFamily); // If this is a new column family, verify that the directory exists - if (wl == null) { + if (wi == null) { Path writerPath = null; if (writeMultipleTables) { Path tableRelPath = getTableRelativePath(tableNameBytes); @@ -310,14 +326,14 @@ public void write(ImmutableBytesWritable row, V cell) throws IOException { // This can only happen once a row is finished though if ( - wl != null && wl.written + length >= maxsize + wi != null && wi.written + length >= maxsize && Bytes.compareTo(this.previousRows.get(family), rowKey) != 0 ) { - rollWriters(wl); + rollWriters(wi); } // create a new WAL writer, if necessary - if (wl == null || wl.writer == null) { + if (wi == null || wi.writer == null) { InetSocketAddress[] favoredNodes = null; if (conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) { HRegionLocation loc = null; @@ -348,14 +364,15 @@ public void write(ImmutableBytesWritable row, V cell) throws IOException { } } } - wl = getNewWriter(tableNameBytes, family, conf, favoredNodes); + wi = getNewWriter(tableNameBytes, family, conf, favoredNodes); } // we now have the proper WAL writer. full steam ahead PrivateCellUtil.updateLatestStamp(cell, this.now); - wl.writer.append(kv); - wl.written += length; + wi.writer.append(kv); + wi.written += length; + wi.maxSequenceId = Math.max(kv.getSequenceId(), wi.maxSequenceId); // Copy the row so we know when a row transition. this.previousRows.put(family, rowKey); @@ -371,24 +388,25 @@ private Path getTableRelativePath(byte[] tableNameBytes) { return tableRelPath; } - private void rollWriters(WriterLength writerLength) throws IOException { - if (writerLength != null) { - closeWriter(writerLength); + private void rollWriters(WriterInfo writerInfo) throws IOException { + if (writerInfo != null) { + closeWriter(writerInfo); } else { - for (WriterLength wl : this.writers.values()) { - closeWriter(wl); + for (WriterInfo wi : this.writers.values()) { + closeWriter(wi); } } } - private void closeWriter(WriterLength wl) throws IOException { - if (wl.writer != null) { + private void closeWriter(WriterInfo wi) throws IOException { + if (wi.writer != null) { LOG.info( - "Writer=" + wl.writer.getPath() + ((wl.written == 0) ? "" : ", wrote=" + wl.written)); - close(wl.writer); - wl.writer = null; + "Writer=" + wi.writer.getPath() + ((wi.written == 0) ? "" : ", wrote=" + wi.written)); + close(wi.writer, wi); + wi.writer = null; } - wl.written = 0; + wi.written = 0; + wi.maxSequenceId = -1; } private Configuration createRemoteClusterConf(Configuration conf) { @@ -428,11 +446,11 @@ private Configuration createRemoteClusterConf(Configuration conf) { /* * Create a new StoreFile.Writer. - * @return A WriterLength, containing a new StoreFile.Writer. + * @return A WriterInfo, containing a new StoreFile.Writer. */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "BX_UNBOXING_IMMEDIATELY_REBOXED", justification = "Not important") - private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration conf, + private WriterInfo getNewWriter(byte[] tableName, byte[] family, Configuration conf, InetSocketAddress[] favoredNodes) throws IOException { byte[] tableAndFamily = getTableNameSuffixedWithFamily(tableName, family); Path familydir = new Path(outputDir, Bytes.toString(family)); @@ -440,7 +458,7 @@ private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration familydir = new Path(outputDir, new Path(getTableRelativePath(tableName), Bytes.toString(family))); } - WriterLength wl = new WriterLength(); + WriterInfo wi = new WriterInfo(); Algorithm compression = overriddenCompression; compression = compression == null ? compressionMap.get(tableAndFamily) : compression; compression = compression == null ? defaultCompression : compression; @@ -467,23 +485,26 @@ private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration HFileContext hFileContext = contextBuilder.build(); if (null == favoredNodes) { - wl.writer = + wi.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs).withOutputDir(familydir) .withBloomType(bloomType).withFileContext(hFileContext).build(); } else { - wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs)) + wi.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs)) .withOutputDir(familydir).withBloomType(bloomType).withFileContext(hFileContext) .withFavoredNodes(favoredNodes).build(); } - this.writers.put(tableAndFamily, wl); - return wl; + this.writers.put(tableAndFamily, wi); + return wi; } - private void close(final StoreFileWriter w) throws IOException { + private void close(final StoreFileWriter w, final WriterInfo wl) throws IOException { if (w != null) { w.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(EnvironmentEdgeManager.currentTime())); w.appendFileInfo(BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString())); + if (conf.getBoolean(SET_MAX_SEQ_ID_KEY, false) && wl.maxSequenceId >= 0) { + w.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(wl.maxSequenceId)); + } w.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude)); w.appendTrackedTimestampsToMetadata(); @@ -493,8 +514,8 @@ private void close(final StoreFileWriter w) throws IOException { @Override public void close(TaskAttemptContext c) throws IOException, InterruptedException { - for (WriterLength wl : this.writers.values()) { - close(wl.writer); + for (WriterInfo wi : this.writers.values()) { + close(wi.writer, wi); } } }; @@ -517,8 +538,9 @@ static void configureStoragePolicy(final Configuration conf, final FileSystem fs /* * Data structure to hold a Writer and amount of data written on it. */ - static class WriterLength { + static class WriterInfo { long written = 0; + long maxSequenceId = -1; StoreFileWriter writer = null; } @@ -579,12 +601,19 @@ private static void writePartitions(Configuration conf, Path partitionsPath, // Write the actual file FileSystem fs = partitionsPath.getFileSystem(conf); - SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, partitionsPath, - ImmutableBytesWritable.class, NullWritable.class); + boolean diskBasedSortingEnabled = diskBasedSortingEnabled(conf); + Class keyClass = + diskBasedSortingEnabled ? KeyOnlyCellComparable.class : ImmutableBytesWritable.class; + SequenceFile.Writer writer = + SequenceFile.createWriter(fs, conf, partitionsPath, keyClass, NullWritable.class); try { for (ImmutableBytesWritable startKey : sorted) { - writer.append(startKey, NullWritable.get()); + Writable writable = diskBasedSortingEnabled + ? new KeyOnlyCellComparable(KeyValueUtil.createFirstOnRow(startKey.get())) + : startKey; + + writer.append(writable, NullWritable.get()); } } finally { writer.close(); @@ -608,7 +637,7 @@ private static void writePartitions(Configuration conf, Path partitionsPath, public static void configureIncrementalLoad(Job job, Table table, RegionLocator regionLocator) throws IOException { configureIncrementalLoad(job, table.getDescriptor(), regionLocator); - configureRemoteCluster(job, table.getConfiguration()); + configureForRemoteCluster(job, table.getConfiguration()); } /** @@ -631,6 +660,10 @@ public static void configureIncrementalLoad(Job job, TableDescriptor tableDescri configureIncrementalLoad(job, singleTableInfo, HFileOutputFormat2.class); } + public static boolean diskBasedSortingEnabled(Configuration conf) { + return conf.getBoolean(DISK_BASED_SORTING_ENABLED_KEY, DISK_BASED_SORTING_ENABLED_DEFAULT); + } + static void configureIncrementalLoad(Job job, List multiTableInfo, Class> cls) throws IOException { Configuration conf = job.getConfiguration(); @@ -652,7 +685,13 @@ static void configureIncrementalLoad(Job job, List multiTableInfo, // Based on the configured map output class, set the correct reducer to properly // sort the incoming values. // TODO it would be nice to pick one or the other of these formats. - if ( + boolean diskBasedSorting = diskBasedSortingEnabled(conf); + + if (diskBasedSorting) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + job.setReducerClass(PreSortedCellsReducer.class); + } else if ( KeyValue.class.equals(job.getMapOutputValueClass()) || MapReduceExtendedCell.class.equals(job.getMapOutputValueClass()) ) { @@ -729,6 +768,11 @@ private static void mergeSerializations(Configuration conf) { // SerializationFactory runs through serializations in the order they are registered. // We want to register ExtendedCellSerialization before CellSerialization because both // work for ExtendedCells but only ExtendedCellSerialization handles them properly. + + if (diskBasedSortingEnabled(conf)) { + serializations.add(OrderPreservedExtendedCellSerialization.class.getName()); + } + if ( conf.getBoolean(EXTENDED_CELL_SERIALIZATION_ENABLED_KEY, EXTENDED_CELL_SERIALIZATION_ENABLED_DEFULT) @@ -786,8 +830,34 @@ public static void configureIncrementalLoadMap(Job job, TableDescriptor tableDes * @see #REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY * @see #REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY * @see #REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY + * @deprecated As of release 2.6.4, this will be removed in HBase 4.0.0 Use + * {@link #configureForRemoteCluster(Job, Configuration)} instead. */ + @Deprecated public static void configureRemoteCluster(Job job, Configuration clusterConf) { + try { + configureForRemoteCluster(job, clusterConf); + } catch (IOException e) { + LOG.error("Configure remote cluster error.", e); + throw new UncheckedIOException("Configure remote cluster error.", e); + } + } + + /** + * Configure HBase cluster key for remote cluster to load region location for locality-sensitive + * if it's enabled. It's not necessary to call this method explicitly when the cluster key for + * HBase cluster to be used to load region location is configured in the job configuration. Call + * this method when another HBase cluster key is configured in the job configuration. For example, + * you should call when you load data from HBase cluster A using {@link TableInputFormat} and + * generate hfiles for HBase cluster B. Otherwise, HFileOutputFormat2 fetch location from cluster + * A and locality-sensitive won't working correctly. If authentication is enabled, it obtains the + * token for the specific cluster. + * @param job which has configuration to be updated + * @param clusterConf which contains cluster key of the HBase cluster to be locality-sensitive + * @throws IOException Exception while initializing cluster credentials + */ + public static void configureForRemoteCluster(Job job, Configuration clusterConf) + throws IOException { Configuration conf = job.getConfiguration(); if (!conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) { @@ -804,6 +874,8 @@ public static void configureRemoteCluster(Job job, Configuration clusterConf) { conf.setInt(REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY, clientPort); conf.set(REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY, parent); + TableMapReduceUtil.initCredentialsForCluster(job, clusterConf); + LOG.info("ZK configs for remote cluster of bulkload is configured: " + quorum + ":" + clientPort + "/" + parent); } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java index 4adcfbfcd3f6..03abcf159753 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java @@ -200,6 +200,10 @@ public CellWritableComparable(Cell kv) { this.kv = kv; } + public Cell getCell() { + return kv; + } + @Override public void write(DataOutput out) throws IOException { int keyLen = PrivateCellUtil.estimatedSerializedSizeOfKey(kv); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java new file mode 100644 index 000000000000..d9b28f8a6895 --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.IOException; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.ExtendedCell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableComparator; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class KeyOnlyCellComparable implements WritableComparable { + + static { + WritableComparator.define(KeyOnlyCellComparable.class, new KeyOnlyCellComparator()); + } + + private ExtendedCell cell = null; + + public KeyOnlyCellComparable() { + } + + public KeyOnlyCellComparable(ExtendedCell cell) { + this.cell = cell; + } + + public ExtendedCell getCell() { + return cell; + } + + @Override + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "EQ_COMPARETO_USE_OBJECT_EQUALS", + justification = "This is wrong, yes, but we should be purging Writables, not fixing them") + public int compareTo(KeyOnlyCellComparable o) { + return CellComparator.getInstance().compare(cell, o.cell); + } + + @Override + public void write(DataOutput out) throws IOException { + int keyLen = PrivateCellUtil.estimatedSerializedSizeOfKey(cell); + int valueLen = 0; // We avoid writing value here. So just serialize as if an empty value. + out.writeInt(keyLen + valueLen + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); + out.writeInt(keyLen); + out.writeInt(valueLen); + PrivateCellUtil.writeFlatKey(cell, out); + out.writeLong(cell.getSequenceId()); + } + + @Override + public void readFields(DataInput in) throws IOException { + cell = KeyValue.create(in); + long seqId = in.readLong(); + cell.setSequenceId(seqId); + } + + public static class KeyOnlyCellComparator extends WritableComparator { + + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + try (DataInputStream d1 = new DataInputStream(new ByteArrayInputStream(b1, s1, l1)); + DataInputStream d2 = new DataInputStream(new ByteArrayInputStream(b2, s2, l2))) { + KeyOnlyCellComparable kv1 = new KeyOnlyCellComparable(); + kv1.readFields(d1); + KeyOnlyCellComparable kv2 = new KeyOnlyCellComparable(); + kv2.readFields(d2); + return compare(kv1, kv2); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/OrderPreservedExtendedCellSerialization.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/OrderPreservedExtendedCellSerialization.java new file mode 100644 index 000000000000..1a7deeff5f9d --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/OrderPreservedExtendedCellSerialization.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.OrderPreservedMapReduceExtendedCell; +import org.apache.hadoop.io.serializer.Deserializer; +import org.apache.hadoop.io.serializer.Serialization; +import org.apache.hadoop.io.serializer.Serializer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class OrderPreservedExtendedCellSerialization + implements Serialization { + + @Override + public boolean accept(Class c) { + return OrderPreservedMapReduceExtendedCell.class.isAssignableFrom(c); + } + + @Override + public Serializer + getSerializer(Class c) { + return new OrderPreservedExtendedCellSerializer(); + } + + @Override + public Deserializer + getDeserializer(Class c) { + return new OrderPreservedExtendedCellDeserializer(); + } + + public static class OrderPreservedExtendedCellSerializer + implements Serializer { + private DataOutputStream dos; + + @Override + public void open(OutputStream os) throws IOException { + this.dos = new DataOutputStream(os); + } + + @Override + public void serialize(OrderPreservedMapReduceExtendedCell kv) throws IOException { + dos.writeInt(PrivateCellUtil.estimatedSerializedSizeOf(kv) - Bytes.SIZEOF_INT); + PrivateCellUtil.writeCell(kv, dos, true); + dos.writeLong(kv.getSequenceId()); + dos.writeInt(kv.getOrder()); + } + + @Override + public void close() throws IOException { + dos.close(); + } + } + + public static class OrderPreservedExtendedCellDeserializer + implements Deserializer { + private DataInputStream dis; + + @Override + public void open(InputStream is) throws IOException { + this.dis = new DataInputStream(is); + } + + @Override + public OrderPreservedMapReduceExtendedCell + deserialize(OrderPreservedMapReduceExtendedCell ignore) throws IOException { + KeyValue kv = KeyValueUtil.create(this.dis); + PrivateCellUtil.setSequenceId(kv, this.dis.readLong()); + int order = dis.readInt(); + return new OrderPreservedMapReduceExtendedCell(kv, order); + } + + @Override + public void close() throws IOException { + dis.close(); + } + } +} diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java new file mode 100644 index 000000000000..113235bfeb42 --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.IOException; +import java.util.Comparator; +import java.util.PriorityQueue; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.OrderPreservedMapReduceExtendedCell; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class PreSortedCellsReducer extends Reducer { + + @Override + protected void reduce(KeyOnlyCellComparable key, + Iterable values, Context context) + throws IOException, InterruptedException { + + PriorityQueue cells = new PriorityQueue<>( + Comparator.comparingInt(OrderPreservedMapReduceExtendedCell::getOrder).reversed()); + + for (OrderPreservedMapReduceExtendedCell cell : values) { + OrderPreservedMapReduceExtendedCell copy = + new OrderPreservedMapReduceExtendedCell(cell.deepClone(), cell.getOrder()); + cells.add(copy); + } + + int index = 0; + for (OrderPreservedMapReduceExtendedCell cell : cells) { + context.write(new ImmutableBytesWritable(CellUtil.cloneRow(key.getCell())), cell); + + if (++index % 100 == 0) { + context.setStatus("Wrote " + index + " cells"); + } + } + } +} diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java index 109aeef4ce0c..54a357f5bef2 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.OutputFormat; @@ -72,6 +73,16 @@ public class TableOutputFormat extends OutputFormat implemen */ public static final String OUTPUT_CONF_PREFIX = "hbase.mapred.output."; + /** + * The configuration key for specifying a custom + * {@link org.apache.hadoop.mapreduce.OutputCommitter} implementation to be used by + * {@link TableOutputFormat}. The value for this property should be the fully qualified class name + * of the custom committer. If this property is not set, {@link TableOutputCommitter} will be used + * by default. + */ + public static final String OUTPUT_COMMITTER_CLASS = + "hbase.mapreduce.tableoutputformat.output.committer.class"; + /** * Optional job parameter to specify a peer cluster. Used specifying remote cluster when copying * between hbase clusters (the source is picked up from hbase-site.xml). @@ -216,7 +227,18 @@ public void checkOutputSpecs(JobContext context) throws IOException, Interrupted @Override public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException { - return new TableOutputCommitter(); + Configuration hConf = getConf(); + if (hConf == null) { + hConf = context.getConfiguration(); + } + + try { + Class outputCommitter = + hConf.getClass(OUTPUT_COMMITTER_CLASS, TableOutputCommitter.class, OutputCommitter.class); + return ReflectionUtils.newInstance(outputCommitter); + } catch (Exception e) { + throw new IOException("Could not create the configured OutputCommitter", e); + } } @Override diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java index 7362f585d319..c4f3187b04dc 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java @@ -71,6 +71,7 @@ static class WALSplit extends InputSplit implements Writable { private long fileSize; private long startTime; private long endTime; + private String[] locations; /** for serialization */ public WALSplit() { @@ -85,6 +86,19 @@ public WALSplit(String logFileName, long fileSize, long startTime, long endTime) this.fileSize = fileSize; this.startTime = startTime; this.endTime = endTime; + this.locations = new String[] {}; + } + + /** + * Represent an WALSplit with location hints for rack-aware processing. + */ + public WALSplit(String logFileName, long fileSize, long startTime, long endTime, + String[] locations) { + this.logFileName = logFileName; + this.fileSize = fileSize; + this.startTime = startTime; + this.endTime = endTime; + this.locations = locations != null ? locations : new String[] {}; } @Override @@ -94,8 +108,7 @@ public long getLength() throws IOException, InterruptedException { @Override public String[] getLocations() throws IOException, InterruptedException { - // TODO: Find the data node with the most blocks for this WAL? - return new String[] {}; + return locations; } public String getLogFileName() { @@ -116,6 +129,11 @@ public void readFields(DataInput in) throws IOException { fileSize = in.readLong(); startTime = in.readLong(); endTime = in.readLong(); + int locationsCount = in.readInt(); + locations = new String[locationsCount]; + for (int i = 0; i < locationsCount; i++) { + locations[i] = in.readUTF(); + } } @Override @@ -124,6 +142,10 @@ public void write(DataOutput out) throws IOException { out.writeLong(fileSize); out.writeLong(startTime); out.writeLong(endTime); + out.writeInt(locations.length); + for (String location : locations) { + out.writeUTF(location); + } } @Override @@ -318,7 +340,7 @@ List getSplits(final JobContext context, final String startKey, fina for (Path inputPath : inputPaths) { FileSystem fs = inputPath.getFileSystem(conf); try { - List files = getFiles(fs, inputPath, startTime, endTime); + List files = getFiles(fs, inputPath, startTime, endTime, conf); allFiles.addAll(files); } catch (FileNotFoundException e) { if (ignoreMissing) { @@ -328,10 +350,25 @@ List getSplits(final JobContext context, final String startKey, fina throw e; } } - List splits = new ArrayList(allFiles.size()); + // Create InputSplits with location hints for each WAL file + Class locationResolverClass = + conf.getClass(WALPlayer.CONF_WAL_FILE_LOCATION_RESOLVER_CLASS, + WALPlayer.NoopWALFileLocationResolver.class, WALPlayer.WALFileLocationResolver.class); + + WALPlayer.WALFileLocationResolver locationResolver = + org.apache.hadoop.util.ReflectionUtils.newInstance(locationResolverClass, conf); + + List splits = new ArrayList<>(); for (FileStatus file : allFiles) { - splits.add(new WALSplit(file.getPath().toString(), file.getLen(), startTime, endTime)); + // Get locations for this specific WAL file + String[] locations = locationResolver + .getLocationsForWALFiles(Collections.singletonList(file.getPath().toString())) + .toArray(new String[0]); + + splits + .add(new WALSplit(file.getPath().toString(), file.getLen(), startTime, endTime, locations)); } + return splits; } @@ -349,11 +386,11 @@ private Path[] getInputPaths(Configuration conf) { * equal to this value else we will filter out the file. If name does not seem to * have a timestamp, we will just return it w/o filtering. */ - private List getFiles(FileSystem fs, Path dir, long startTime, long endTime) - throws IOException { + private List getFiles(FileSystem fs, Path dir, long startTime, long endTime, + Configuration conf) throws IOException { List result = new ArrayList<>(); LOG.debug("Scanning " + dir.toString() + " for WAL files"); - RemoteIterator iter = fs.listLocatedStatus(dir); + RemoteIterator iter = listLocatedFileStatus(fs, dir, conf); if (!iter.hasNext()) { return Collections.emptyList(); } @@ -361,7 +398,7 @@ private List getFiles(FileSystem fs, Path dir, long startTime, long LocatedFileStatus file = iter.next(); if (file.isDirectory()) { // Recurse into sub directories - result.addAll(getFiles(fs, file.getPath(), startTime, endTime)); + result.addAll(getFiles(fs, file.getPath(), startTime, endTime, conf)); } else { addFile(result, file, startTime, endTime); } @@ -396,4 +433,28 @@ public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { return new WALKeyRecordReader(); } + + /** + * Attempts to return the {@link LocatedFileStatus} for the given directory. If the directory does + * not exist, it will check if the directory is an archived log file and try to find it + */ + private static RemoteIterator listLocatedFileStatus(FileSystem fs, Path dir, + Configuration conf) throws IOException { + try { + return fs.listLocatedStatus(dir); + } catch (FileNotFoundException e) { + if (AbstractFSWALProvider.isArchivedLogFile(dir)) { + throw e; + } + + LOG.warn("Log file {} not found, trying to find it in archive directory.", dir); + Path archiveFile = AbstractFSWALProvider.findArchivedLog(dir, conf); + if (archiveFile == null) { + LOG.error("Did not find archive file for {}", dir); + throw e; + } + + return fs.listLocatedStatus(archiveFile); + } + } } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index e06300848f68..ce490fb420e0 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -21,6 +21,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -32,6 +33,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; @@ -51,8 +53,10 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.MapReduceExtendedCell; +import org.apache.hadoop.hbase.util.OrderPreservedMapReduceExtendedCell; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; @@ -63,6 +67,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + /** * A tool to replay WAL files as a M/R job. The WAL can be replayed for a set of tables or all * tables, and a time range can be provided (in milliseconds). The WAL is filtered to the passed set @@ -85,6 +91,27 @@ public class WALPlayer extends Configured implements Tool { private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; + // Configuration key for pluggable WAL location resolver class name + // Used to enable rack-aware processing by providing preferred data locality hints for WAL files + public static final String CONF_WAL_FILE_LOCATION_RESOLVER_CLASS = + "wal.backup.file.location.resolver.class"; + + /** + * Interface for resolving file locations to influence InputSplit placement for rack-aware WAL + * processing. + */ + @InterfaceAudience.Public + public interface WALFileLocationResolver { + Set getLocationsForWALFiles(final Collection walFiles); + } + + public static class NoopWALFileLocationResolver implements WALFileLocationResolver { + @Override + public Set getLocationsForWALFiles(Collection walFiles) { + return ImmutableSet.of(); + } + } + public WALPlayer() { } @@ -138,15 +165,17 @@ public void setup(Context context) throws IOException { /** * A mapper that just writes out Cells. This one can be used together with {@link CellSortReducer} */ - static class WALCellMapper extends Mapper { + static class WALCellMapper extends Mapper, Cell> { private Set tableSet = new HashSet<>(); private boolean multiTableSupport = false; + private boolean diskBasedSortingEnabled = false; @Override public void map(WALKey key, WALEdit value, Context context) throws IOException { try { TableName table = key.getTableName(); if (tableSet.contains(table.getNameAsString())) { + int order = 0; for (Cell cell : value.getCells()) { if (WALEdit.isMetaEditFamily(cell)) { continue; @@ -161,7 +190,9 @@ public void map(WALKey key, WALEdit value, Context context) throws IOException { byte[] outKey = multiTableSupport ? Bytes.add(table.getName(), Bytes.toBytes(tableSeparator), CellUtil.cloneRow(cell)) : CellUtil.cloneRow(cell); - context.write(new ImmutableBytesWritable(outKey), new MapReduceExtendedCell(cell)); + ExtendedCell extendedCell = (ExtendedCell) cell; + context.write(wrapKey(outKey, extendedCell), wrapCell(extendedCell, order)); + ++order; } } } catch (InterruptedException e) { @@ -174,8 +205,30 @@ public void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); String[] tables = conf.getStrings(TABLES_KEY); this.multiTableSupport = conf.getBoolean(MULTI_TABLES_SUPPORT, false); + this.diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); Collections.addAll(tableSet, tables); } + + private MapReduceExtendedCell wrapCell(ExtendedCell cell, int order) { + if (this.diskBasedSortingEnabled) { + return new OrderPreservedMapReduceExtendedCell(cell, order); + } + return new MapReduceExtendedCell(cell); + } + + private WritableComparable wrapKey(byte[] key, ExtendedCell cell) { + if (this.diskBasedSortingEnabled) { + // Important to build a new cell with the updated key to maintain multi-table support + KeyValue kv = new KeyValue(key, 0, key.length, cell.getFamilyArray(), + cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), + cell.getQualifierOffset(), cell.getQualifierLength(), cell.getTimestamp(), + KeyValue.Type.codeToType(cell.getTypeByte()), null, 0, 0); + kv.setSequenceId(cell.getSequenceId()); + return new KeyOnlyCellComparable(kv); + } else { + return new ImmutableBytesWritable(key); + } + } } /** @@ -353,13 +406,19 @@ public Job createSubmittableJob(String[] args) throws IOException { job.setJarByClass(WALPlayer.class); job.setInputFormatClass(WALInputFormat.class); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); + boolean diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); + if (diskBasedSortingEnabled) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + } else { + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + } String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); if (hfileOutPath != null) { LOG.debug("add incremental job :" + hfileOutPath + " from " + inputDirs); - if (!multiTableSupport && tables.length != 1) { + if (!multiTableSupport && tableMap.length != 1) { throw new IOException("Exactly one table must be specified for the bulk export option"); } @@ -369,13 +428,18 @@ public Job createSubmittableJob(String[] args) throws IOException { true); // the bulk HFile case - List tableNames = getTableNameList(tables); + List tableNames = getTableNameList(tableMap); job.setMapperClass(WALCellMapper.class); - job.setReducerClass(CellSortReducer.class); + if (diskBasedSortingEnabled) { + job.setReducerClass(PreSortedCellsReducer.class); + job.setMapOutputValueClass(OrderPreservedMapReduceExtendedCell.class); + } else { + job.setMapOutputValueClass(MapReduceExtendedCell.class); + job.setReducerClass(CellSortReducer.class); + } Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputValueClass(MapReduceExtendedCell.class); try (Connection conn = ConnectionFactory.createConnection(conf);) { List tableInfoList = new ArrayList<>(); for (TableName tableName : tableNames) { diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index 9616bb1605a2..ca35cb5b273d 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -325,6 +325,8 @@ public void map(BytesWritable key, NullWritable value, Context context) Path outputPath = getOutputPath(inputInfo); copyFile(context, inputInfo, outputPath); + // inject failure + injectTestFailure(context, inputInfo); } /** @@ -351,19 +353,23 @@ private Path getOutputPath(final SnapshotFileInfo inputInfo) throws IOException return new Path(outputArchive, path); } - @SuppressWarnings("checkstyle:linelength") /** * Used by TestExportSnapshot to test for retries when failures happen. Failure is injected in - * {@link #copyFile(Mapper.Context, org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo, Path)}. + * {@link #map(BytesWritable, NullWritable, org.apache.hadoop.mapreduce.Mapper.Context)} */ private void injectTestFailure(final Context context, final SnapshotFileInfo inputInfo) throws IOException { - if (!context.getConfiguration().getBoolean(Testing.CONF_TEST_FAILURE, false)) return; - if (testing.injectedFailureCount >= testing.failuresCountToInject) return; + if (!context.getConfiguration().getBoolean(Testing.CONF_TEST_FAILURE, false)) { + return; + } + if (testing.injectedFailureCount >= testing.failuresCountToInject) { + return; + } testing.injectedFailureCount++; context.getCounter(Counter.COPY_FAILED).increment(1); LOG.debug("Injecting failure. Count: " + testing.injectedFailureCount); - throw new IOException(String.format("TEST FAILURE (%d of max %d): Unable to copy input=%s", + throw new IOException(String.format( + context.getTaskAttemptID() + " TEST FAILURE (%d of max %d): Unable to copy input=%s", testing.injectedFailureCount, testing.failuresCountToInject, inputInfo)); } @@ -426,8 +432,6 @@ private void copyFile(final Context context, final SnapshotFileInfo inputInfo, LOG.error("Error copying " + inputPath + " to " + outputPath, e); context.getCounter(Counter.COPY_FAILED).increment(1); throw e; - } finally { - injectTestFailure(context, inputInfo); } } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/OrderPreservedMapReduceExtendedCell.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/OrderPreservedMapReduceExtendedCell.java new file mode 100644 index 000000000000..7154af70245a --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/OrderPreservedMapReduceExtendedCell.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.Cell; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class OrderPreservedMapReduceExtendedCell extends MapReduceExtendedCell { + + private final int order; + + public OrderPreservedMapReduceExtendedCell(Cell cell, int order) { + super(cell); + this.order = order; + } + + public int getOrder() { + return order; + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java index 283acbabf6e4..d6c4b623ad42 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.util.ToolRunner; @@ -172,7 +173,7 @@ private void testWALKeyValueMapper(final String tableConfigKey) throws Exception WALKey key = mock(WALKey.class); when(key.getTableName()).thenReturn(TableName.valueOf("table")); @SuppressWarnings("unchecked") - Mapper.Context context = mock(Context.class); + Mapper, Cell>.Context context = mock(Context.class); when(context.getConfiguration()).thenReturn(configuration); WALEdit value = mock(WALEdit.class); diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java new file mode 100644 index 000000000000..b4cb6a8355fc --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.Closeable; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for {@link HFileOutputFormat2} with secure mode. + */ +@Category({ VerySlowMapReduceTests.class, LargeTests.class }) +public class TestHFileOutputFormat2WithSecurity { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestHFileOutputFormat2WithSecurity.class); + + private static final byte[] FAMILIES = Bytes.toBytes("test_cf"); + + private static final String HTTP_PRINCIPAL = "HTTP/localhost"; + + private HBaseTestingUtility utilA; + + private Configuration confA; + + private HBaseTestingUtility utilB; + + private MiniKdc kdc; + + private List clusters = new ArrayList<>(); + + @Before + public void setupSecurityClusters() throws Exception { + utilA = new HBaseTestingUtility(); + confA = utilA.getConfiguration(); + + utilB = new HBaseTestingUtility(); + + // Prepare security configs. + File keytab = new File(utilA.getDataTestDir("keytab").toUri().getPath()); + kdc = utilA.setupMiniKdc(keytab); + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + // Start security clusterA + clusters.add(utilA.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)); + + // Start security clusterB + clusters.add(utilB.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)); + } + + @After + public void teardownSecurityClusters() { + IOUtils.closeQuietly(clusters); + clusters.clear(); + if (kdc != null) { + kdc.stop(); + } + } + + @Test + public void testIncrementalLoadInMultiClusterWithSecurity() throws Exception { + TableName tableName = TableName.valueOf("testIncrementalLoadInMultiClusterWithSecurity"); + + // Create table in clusterB + try (Table table = utilB.createTable(tableName, FAMILIES); + RegionLocator r = utilB.getConnection().getRegionLocator(tableName)) { + + // Create job in clusterA + Job job = Job.getInstance(confA, "testIncrementalLoadInMultiClusterWithSecurity"); + job.setWorkingDirectory( + utilA.getDataTestDirOnTestFS("testIncrementalLoadInMultiClusterWithSecurity")); + job.setInputFormatClass(NMapInputFormat.class); + job.setMapperClass(TestHFileOutputFormat2.RandomKVGeneratingMapper.class); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(KeyValue.class); + HFileOutputFormat2.configureIncrementalLoad(job, table, r); + + assertEquals(2, job.getCredentials().getAllTokens().size()); + + String remoteClusterId = utilB.getHBaseClusterInterface().getClusterMetrics().getClusterId(); + assertTrue(job.getCredentials().getToken(new Text(remoteClusterId)) != null); + } finally { + if (utilB.getAdmin().tableExists(tableName)) { + utilB.deleteTable(tableName); + } + } + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java index 197502655467..26741f6f4d5e 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java @@ -17,977 +17,26 @@ */ package org.apache.hadoop.hbase.mapreduce; -import static org.apache.hadoop.hbase.HConstants.RPC_CODEC_CONF_KEY; -import static org.apache.hadoop.hbase.ipc.RpcClient.DEFAULT_CODEC_CLASS; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.ArrayBackedTag; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellScanner; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeepDeletedCells; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.PrivateCellUtil; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Durability; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.RegionObserver; -import org.apache.hadoop.hbase.filter.Filter; -import org.apache.hadoop.hbase.filter.FilterBase; -import org.apache.hadoop.hbase.filter.PrefixFilter; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.Import.KeyValueImporter; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; -import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.LauncherSecurityManager; -import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.mapreduce.Mapper.Context; -import org.apache.hadoop.util.GenericOptionsParser; -import org.apache.hadoop.util.ToolRunner; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; /** * Tests the table import and table export MR job functionality */ -@Category({ VerySlowMapReduceTests.class, MediumTests.class }) -// TODO : Remove this in 3.0 -public class TestImportExport { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestImportExport.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestImportExport.class); - protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static final byte[] ROW1 = Bytes.toBytesBinary("\\x32row1"); - private static final byte[] ROW2 = Bytes.toBytesBinary("\\x32row2"); - private static final byte[] ROW3 = Bytes.toBytesBinary("\\x32row3"); - private static final String FAMILYA_STRING = "a"; - private static final String FAMILYB_STRING = "b"; - private static final byte[] FAMILYA = Bytes.toBytes(FAMILYA_STRING); - private static final byte[] FAMILYB = Bytes.toBytes(FAMILYB_STRING); - private static final byte[] QUAL = Bytes.toBytes("q"); - private static final String OUTPUT_DIR = "outputdir"; - private static String FQ_OUTPUT_DIR; - private static final String EXPORT_BATCH_SIZE = "100"; - - private static final long now = EnvironmentEdgeManager.currentTime(); - private final TableName EXPORT_TABLE = TableName.valueOf("export_table"); - private final TableName IMPORT_TABLE = TableName.valueOf("import_table"); - public static final byte TEST_TAG_TYPE = (byte) (Tag.CUSTOM_TAG_TYPE_RANGE + 1); - public static final String TEST_ATTR = "source_op"; - public static final String TEST_TAG = "test_tag"; +@Tag(VerySlowMapReduceTests.TAG) +@Tag(LargeTests.TAG) +public class TestImportExport extends TestImportExportBase { - @BeforeClass + @BeforeAll public static void beforeClass() throws Throwable { - // Up the handlers; this test needs more than usual. - UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); - UTIL.startMiniCluster(); - FQ_OUTPUT_DIR = - new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString(); + TestImportExportBase.setUpBeforeClass(); } - @AfterClass + @AfterAll public static void afterClass() throws Throwable { UTIL.shutdownMiniCluster(); } - - @Rule - public final TestName name = new TestName(); - - @Before - public void announce() { - LOG.info("Running " + name.getMethodName()); - } - - @After - public void cleanup() throws Throwable { - FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - fs.delete(new Path(OUTPUT_DIR), true); - if (UTIL.getAdmin().tableExists(EXPORT_TABLE)) { - UTIL.deleteTable(EXPORT_TABLE); - } - if (UTIL.getAdmin().tableExists(IMPORT_TABLE)) { - UTIL.deleteTable(IMPORT_TABLE); - } - } - - /** - * Runs an export job with the specified command line args - * @return true if job completed successfully - */ - protected boolean runExport(String[] args) throws Throwable { - // need to make a copy of the configuration because to make sure different temp dirs are used. - int status = ToolRunner.run(new Configuration(UTIL.getConfiguration()), new Export(), args); - return status == 0; - } - - protected void runExportMain(String[] args) throws Throwable { - Export.main(args); - } - - /** - * Runs an import job with the specified command line args - * @return true if job completed successfully - */ - boolean runImport(String[] args) throws Throwable { - // need to make a copy of the configuration because to make sure different temp dirs are used. - int status = ToolRunner.run(new Configuration(UTIL.getConfiguration()), new Import(), args); - return status == 0; - } - - /** - * Test simple replication case with column mapping - */ - @Test - public void testSimpleCase() throws Throwable { - try (Table t = UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILYA, 3);) { - Put p = new Put(ROW1); - p.addColumn(FAMILYA, QUAL, now, QUAL); - p.addColumn(FAMILYA, QUAL, now + 1, QUAL); - p.addColumn(FAMILYA, QUAL, now + 2, QUAL); - t.put(p); - p = new Put(ROW2); - p.addColumn(FAMILYA, QUAL, now, QUAL); - p.addColumn(FAMILYA, QUAL, now + 1, QUAL); - p.addColumn(FAMILYA, QUAL, now + 2, QUAL); - t.put(p); - p = new Put(ROW3); - p.addColumn(FAMILYA, QUAL, now, QUAL); - p.addColumn(FAMILYA, QUAL, now + 1, QUAL); - p.addColumn(FAMILYA, QUAL, now + 2, QUAL); - t.put(p); - } - - String[] args = new String[] { - // Only export row1 & row2. - "-D" + TableInputFormat.SCAN_ROW_START + "=\\x32row1", - "-D" + TableInputFormat.SCAN_ROW_STOP + "=\\x32row3", name.getMethodName(), FQ_OUTPUT_DIR, - "1000", // max number of key versions per key to export - }; - assertTrue(runExport(args)); - - final String IMPORT_TABLE = name.getMethodName() + "import"; - try (Table t = UTIL.createTable(TableName.valueOf(IMPORT_TABLE), FAMILYB, 3);) { - args = - new String[] { "-D" + Import.CF_RENAME_PROP + "=" + FAMILYA_STRING + ":" + FAMILYB_STRING, - IMPORT_TABLE, FQ_OUTPUT_DIR }; - assertTrue(runImport(args)); - - Get g = new Get(ROW1); - g.setMaxVersions(); - Result r = t.get(g); - assertEquals(3, r.size()); - g = new Get(ROW2); - g.setMaxVersions(); - r = t.get(g); - assertEquals(3, r.size()); - g = new Get(ROW3); - r = t.get(g); - assertEquals(0, r.size()); - } - } - - /** - * Test export hbase:meta table - */ - @Test - public void testMetaExport() throws Throwable { - String[] args = - new String[] { TableName.META_TABLE_NAME.getNameAsString(), FQ_OUTPUT_DIR, "1", "0", "0" }; - assertTrue(runExport(args)); - } - - /** - * Test import data from 0.94 exported file - */ - @Test - public void testImport94Table() throws Throwable { - final String name = "exportedTableIn94Format"; - URL url = TestImportExport.class.getResource(name); - File f = new File(url.toURI()); - if (!f.exists()) { - LOG.warn("FAILED TO FIND " + f + "; skipping out on test"); - return; - } - assertTrue(f.exists()); - LOG.info("FILE=" + f); - Path importPath = new Path(f.toURI()); - FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - fs.copyFromLocalFile(importPath, new Path(FQ_OUTPUT_DIR + Path.SEPARATOR + name)); - String IMPORT_TABLE = name; - try (Table t = UTIL.createTable(TableName.valueOf(IMPORT_TABLE), Bytes.toBytes("f1"), 3);) { - String[] args = new String[] { "-Dhbase.import.version=0.94", IMPORT_TABLE, FQ_OUTPUT_DIR }; - assertTrue(runImport(args)); - // @formatter:off - // exportedTableIn94Format contains 5 rows - // ROW COLUMN+CELL - // r1 column=f1:c1, timestamp=1383766761171, value=val1 - // r2 column=f1:c1, timestamp=1383766771642, value=val2 - // r3 column=f1:c1, timestamp=1383766777615, value=val3 - // r4 column=f1:c1, timestamp=1383766785146, value=val4 - // r5 column=f1:c1, timestamp=1383766791506, value=val5 - // @formatter:on - assertEquals(5, UTIL.countRows(t)); - } - } - - /** - * Test export scanner batching - */ - @Test - public void testExportScannerBatching() throws Throwable { - TableDescriptor desc = TableDescriptorBuilder - .newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(1).build()) - .build(); - UTIL.getAdmin().createTable(desc); - try (Table t = UTIL.getConnection().getTable(desc.getTableName());) { - - Put p = new Put(ROW1); - p.addColumn(FAMILYA, QUAL, now, QUAL); - p.addColumn(FAMILYA, QUAL, now + 1, QUAL); - p.addColumn(FAMILYA, QUAL, now + 2, QUAL); - p.addColumn(FAMILYA, QUAL, now + 3, QUAL); - p.addColumn(FAMILYA, QUAL, now + 4, QUAL); - t.put(p); - // added scanner batching arg. - String[] args = new String[] { "-D" + ExportUtils.EXPORT_BATCHING + "=" + EXPORT_BATCH_SIZE, - name.getMethodName(), FQ_OUTPUT_DIR }; - assertTrue(runExport(args)); - - FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - fs.delete(new Path(FQ_OUTPUT_DIR), true); - } - } - - @Test - public void testWithDeletes() throws Throwable { - TableDescriptor desc = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5) - .setKeepDeletedCells(KeepDeletedCells.TRUE).build()) - .build(); - UTIL.getAdmin().createTable(desc); - try (Table t = UTIL.getConnection().getTable(desc.getTableName());) { - - Put p = new Put(ROW1); - p.addColumn(FAMILYA, QUAL, now, QUAL); - p.addColumn(FAMILYA, QUAL, now + 1, QUAL); - p.addColumn(FAMILYA, QUAL, now + 2, QUAL); - p.addColumn(FAMILYA, QUAL, now + 3, QUAL); - p.addColumn(FAMILYA, QUAL, now + 4, QUAL); - t.put(p); - - Delete d = new Delete(ROW1, now + 3); - t.delete(d); - d = new Delete(ROW1); - d.addColumns(FAMILYA, QUAL, now + 2); - t.delete(d); - } - - String[] args = new String[] { "-D" + ExportUtils.RAW_SCAN + "=true", name.getMethodName(), - FQ_OUTPUT_DIR, "1000", // max number of key versions per key to export - }; - assertTrue(runExport(args)); - - final String IMPORT_TABLE = name.getMethodName() + "import"; - desc = TableDescriptorBuilder - .newBuilder(TableName.valueOf(IMPORT_TABLE)).setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(FAMILYA).setMaxVersions(5).setKeepDeletedCells(KeepDeletedCells.TRUE).build()) - .build(); - UTIL.getAdmin().createTable(desc); - try (Table t = UTIL.getConnection().getTable(desc.getTableName());) { - args = new String[] { IMPORT_TABLE, FQ_OUTPUT_DIR }; - assertTrue(runImport(args)); - - Scan s = new Scan(); - s.setMaxVersions(); - s.setRaw(true); - ResultScanner scanner = t.getScanner(s); - Result r = scanner.next(); - Cell[] res = r.rawCells(); - assertTrue(PrivateCellUtil.isDeleteFamily(res[0])); - assertEquals(now + 4, res[1].getTimestamp()); - assertEquals(now + 3, res[2].getTimestamp()); - assertTrue(CellUtil.isDelete(res[3])); - assertEquals(now + 2, res[4].getTimestamp()); - assertEquals(now + 1, res[5].getTimestamp()); - assertEquals(now, res[6].getTimestamp()); - } - } - - @Test - public void testWithMultipleDeleteFamilyMarkersOfSameRowSameFamily() throws Throwable { - final TableName exportTable = TableName.valueOf(name.getMethodName()); - TableDescriptor desc = - TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5) - .setKeepDeletedCells(KeepDeletedCells.TRUE).build()) - .build(); - UTIL.getAdmin().createTable(desc); - - Table exportT = UTIL.getConnection().getTable(exportTable); - - // Add first version of QUAL - Put p = new Put(ROW1); - p.addColumn(FAMILYA, QUAL, now, QUAL); - exportT.put(p); - - // Add Delete family marker - Delete d = new Delete(ROW1, now + 3); - exportT.delete(d); - - // Add second version of QUAL - p = new Put(ROW1); - p.addColumn(FAMILYA, QUAL, now + 5, "s".getBytes()); - exportT.put(p); - - // Add second Delete family marker - d = new Delete(ROW1, now + 7); - exportT.delete(d); - - String[] args = new String[] { "-D" + ExportUtils.RAW_SCAN + "=true", - exportTable.getNameAsString(), FQ_OUTPUT_DIR, "1000", // max number of key versions per key to - // export - }; - assertTrue(runExport(args)); - - final String importTable = name.getMethodName() + "import"; - desc = TableDescriptorBuilder - .newBuilder(TableName.valueOf(importTable)).setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(FAMILYA).setMaxVersions(5).setKeepDeletedCells(KeepDeletedCells.TRUE).build()) - .build(); - UTIL.getAdmin().createTable(desc); - - Table importT = UTIL.getConnection().getTable(TableName.valueOf(importTable)); - args = new String[] { importTable, FQ_OUTPUT_DIR }; - assertTrue(runImport(args)); - - Scan s = new Scan(); - s.setMaxVersions(); - s.setRaw(true); - - ResultScanner importedTScanner = importT.getScanner(s); - Result importedTResult = importedTScanner.next(); - - ResultScanner exportedTScanner = exportT.getScanner(s); - Result exportedTResult = exportedTScanner.next(); - try { - Result.compareResults(exportedTResult, importedTResult); - } catch (Throwable e) { - fail("Original and imported tables data comparision failed with error:" + e.getMessage()); - } finally { - exportT.close(); - importT.close(); - } - } - - /** - * Create a simple table, run an Export Job on it, Import with filtering on, verify counts, - * attempt with invalid values. - */ - @Test - public void testWithFilter() throws Throwable { - // Create simple table to export - TableDescriptor desc = TableDescriptorBuilder - .newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5).build()) - .build(); - UTIL.getAdmin().createTable(desc); - Table exportTable = UTIL.getConnection().getTable(desc.getTableName()); - - Put p1 = new Put(ROW1); - p1.addColumn(FAMILYA, QUAL, now, QUAL); - p1.addColumn(FAMILYA, QUAL, now + 1, QUAL); - p1.addColumn(FAMILYA, QUAL, now + 2, QUAL); - p1.addColumn(FAMILYA, QUAL, now + 3, QUAL); - p1.addColumn(FAMILYA, QUAL, now + 4, QUAL); - - // Having another row would actually test the filter. - Put p2 = new Put(ROW2); - p2.addColumn(FAMILYA, QUAL, now, QUAL); - - exportTable.put(Arrays.asList(p1, p2)); - - // Export the simple table - String[] args = new String[] { name.getMethodName(), FQ_OUTPUT_DIR, "1000" }; - assertTrue(runExport(args)); - - // Import to a new table - final String IMPORT_TABLE = name.getMethodName() + "import"; - desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(IMPORT_TABLE)) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5).build()) - .build(); - UTIL.getAdmin().createTable(desc); - - Table importTable = UTIL.getConnection().getTable(desc.getTableName()); - args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(), - "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, FQ_OUTPUT_DIR, - "1000" }; - assertTrue(runImport(args)); - - // get the count of the source table for that time range - PrefixFilter filter = new PrefixFilter(ROW1); - int count = getCount(exportTable, filter); - - Assert.assertEquals("Unexpected row count between export and import tables", count, - getCount(importTable, null)); - - // and then test that a broken command doesn't bork everything - easier here because we don't - // need to re-run the export job - - args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(), - "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", name.getMethodName(), - FQ_OUTPUT_DIR, "1000" }; - assertFalse(runImport(args)); - - // cleanup - exportTable.close(); - importTable.close(); - } - - /** - * Create a simple table, run an Export Job on it, Import with bulk output and enable largeResult - */ - @Test - public void testBulkImportAndLargeResult() throws Throwable { - // Create simple table to export - TableDescriptor desc = TableDescriptorBuilder - .newBuilder(TableName.valueOf(name.getMethodName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5).build()) - .build(); - UTIL.getAdmin().createTable(desc); - Table exportTable = UTIL.getConnection().getTable(desc.getTableName()); - - Put p1 = new Put(ROW1); - p1.addColumn(FAMILYA, QUAL, now, QUAL); - - // Having another row would actually test the filter. - Put p2 = new Put(ROW2); - p2.addColumn(FAMILYA, QUAL, now, QUAL); - - exportTable.put(Arrays.asList(p1, p2)); - - // Export the simple table - String[] args = new String[] { name.getMethodName(), FQ_OUTPUT_DIR, "1000" }; - assertTrue(runExport(args)); - - // Import to a new table - final String IMPORT_TABLE = name.getMethodName() + "import"; - desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(IMPORT_TABLE)) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5).build()) - .build(); - UTIL.getAdmin().createTable(desc); - - String O_OUTPUT_DIR = - new Path(OUTPUT_DIR + 1).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString(); - - args = new String[] { "-D" + Import.BULK_OUTPUT_CONF_KEY + "=" + O_OUTPUT_DIR, - "-D" + Import.HAS_LARGE_RESULT + "=" + true, IMPORT_TABLE, FQ_OUTPUT_DIR, "1000" }; - assertTrue(runImport(args)); - } - - /** - * Count the number of keyvalues in the specified table with the given filter - * @param table the table to scan - * @return the number of keyvalues found - */ - private int getCount(Table table, Filter filter) throws IOException { - Scan scan = new Scan(); - scan.setFilter(filter); - ResultScanner results = table.getScanner(scan); - int count = 0; - for (Result res : results) { - count += res.size(); - } - results.close(); - return count; - } - - /** - * test main method. Import should print help and call System.exit - */ - @Test - public void testImportMain() throws Throwable { - PrintStream oldPrintStream = System.err; - SecurityManager SECURITY_MANAGER = System.getSecurityManager(); - LauncherSecurityManager newSecurityManager = new LauncherSecurityManager(); - System.setSecurityManager(newSecurityManager); - ByteArrayOutputStream data = new ByteArrayOutputStream(); - String[] args = {}; - System.setErr(new PrintStream(data)); - try { - System.setErr(new PrintStream(data)); - Import.main(args); - fail("should be SecurityException"); - } catch (SecurityException e) { - assertEquals(-1, newSecurityManager.getExitCode()); - assertTrue(data.toString().contains("Wrong number of arguments:")); - assertTrue(data.toString().contains("-Dimport.bulk.output=/path/for/output")); - assertTrue(data.toString().contains("-Dimport.filter.class=")); - assertTrue(data.toString().contains("-Dimport.bulk.output=/path/for/output")); - assertTrue(data.toString().contains("-Dmapreduce.reduce.speculative=false")); - } finally { - System.setErr(oldPrintStream); - System.setSecurityManager(SECURITY_MANAGER); - } - } - - @Test - public void testExportScan() throws Exception { - int version = 100; - long startTime = EnvironmentEdgeManager.currentTime(); - long endTime = startTime + 1; - String prefix = "row"; - String label_0 = "label_0"; - String label_1 = "label_1"; - String[] args = { "table", "outputDir", String.valueOf(version), String.valueOf(startTime), - String.valueOf(endTime), prefix }; - Scan scan = ExportUtils.getScanFromCommandLine(UTIL.getConfiguration(), args); - assertEquals(version, scan.getMaxVersions()); - assertEquals(startTime, scan.getTimeRange().getMin()); - assertEquals(endTime, scan.getTimeRange().getMax()); - assertEquals(true, (scan.getFilter() instanceof PrefixFilter)); - assertEquals(0, - Bytes.compareTo(((PrefixFilter) scan.getFilter()).getPrefix(), Bytes.toBytesBinary(prefix))); - String[] argsWithLabels = - { "-D " + ExportUtils.EXPORT_VISIBILITY_LABELS + "=" + label_0 + "," + label_1, "table", - "outputDir", String.valueOf(version), String.valueOf(startTime), String.valueOf(endTime), - prefix }; - Configuration conf = new Configuration(UTIL.getConfiguration()); - // parse the "-D" options - String[] otherArgs = new GenericOptionsParser(conf, argsWithLabels).getRemainingArgs(); - Scan scanWithLabels = ExportUtils.getScanFromCommandLine(conf, otherArgs); - assertEquals(version, scanWithLabels.getMaxVersions()); - assertEquals(startTime, scanWithLabels.getTimeRange().getMin()); - assertEquals(endTime, scanWithLabels.getTimeRange().getMax()); - assertEquals(true, (scanWithLabels.getFilter() instanceof PrefixFilter)); - assertEquals(0, Bytes.compareTo(((PrefixFilter) scanWithLabels.getFilter()).getPrefix(), - Bytes.toBytesBinary(prefix))); - assertEquals(2, scanWithLabels.getAuthorizations().getLabels().size()); - assertEquals(label_0, scanWithLabels.getAuthorizations().getLabels().get(0)); - assertEquals(label_1, scanWithLabels.getAuthorizations().getLabels().get(1)); - } - - /** - * test main method. Export should print help and call System.exit - */ - @Test - public void testExportMain() throws Throwable { - PrintStream oldPrintStream = System.err; - SecurityManager SECURITY_MANAGER = System.getSecurityManager(); - LauncherSecurityManager newSecurityManager = new LauncherSecurityManager(); - System.setSecurityManager(newSecurityManager); - ByteArrayOutputStream data = new ByteArrayOutputStream(); - String[] args = {}; - System.setErr(new PrintStream(data)); - try { - System.setErr(new PrintStream(data)); - runExportMain(args); - fail("should be SecurityException"); - } catch (SecurityException e) { - assertEquals(-1, newSecurityManager.getExitCode()); - String errMsg = data.toString(); - assertTrue(errMsg.contains("Wrong number of arguments:")); - assertTrue( - errMsg.contains("Usage: Export [-D ]* [ " - + "[ []] [^[regex pattern] or [Prefix] to filter]]")); - assertTrue(errMsg.contains("-D hbase.mapreduce.scan.column.family=,, ...")); - assertTrue(errMsg.contains("-D hbase.mapreduce.include.deleted.rows=true")); - assertTrue(errMsg.contains("-D hbase.client.scanner.caching=100")); - assertTrue(errMsg.contains("-D hbase.export.scanner.batch=10")); - assertTrue(errMsg.contains("-D hbase.export.scanner.caching=100")); - } finally { - System.setErr(oldPrintStream); - System.setSecurityManager(SECURITY_MANAGER); - } - } - - /** - * Test map method of Importer - */ - @SuppressWarnings({ "unchecked", "rawtypes" }) - @Test - public void testKeyValueImporter() throws Throwable { - KeyValueImporter importer = new KeyValueImporter(); - Configuration configuration = new Configuration(); - Context ctx = mock(Context.class); - when(ctx.getConfiguration()).thenReturn(configuration); - - doAnswer(new Answer() { - - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - ImmutableBytesWritable writer = invocation.getArgument(0); - KeyValue key = invocation.getArgument(1); - assertEquals("Key", Bytes.toString(writer.get())); - assertEquals("row", Bytes.toString(CellUtil.cloneRow(key))); - return null; - } - }).when(ctx).write(any(), any()); - - importer.setup(ctx); - Result value = mock(Result.class); - KeyValue[] keys = { - new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"), - Bytes.toBytes("value")), - new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"), - Bytes.toBytes("value1")) }; - when(value.rawCells()).thenReturn(keys); - importer.map(new ImmutableBytesWritable(Bytes.toBytes("Key")), value, ctx); - - } - - /** - * Test addFilterAndArguments method of Import This method set couple parameters into - * Configuration - */ - @Test - public void testAddFilterAndArguments() throws IOException { - Configuration configuration = new Configuration(); - - List args = new ArrayList<>(); - args.add("param1"); - args.add("param2"); - - Import.addFilterAndArguments(configuration, FilterBase.class, args); - assertEquals("org.apache.hadoop.hbase.filter.FilterBase", - configuration.get(Import.FILTER_CLASS_CONF_KEY)); - assertEquals("param1,param2", configuration.get(Import.FILTER_ARGS_CONF_KEY)); - } - - @Test - public void testDurability() throws Throwable { - // Create an export table. - String exportTableName = name.getMethodName() + "export"; - try (Table exportTable = UTIL.createTable(TableName.valueOf(exportTableName), FAMILYA, 3);) { - - // Insert some data - Put put = new Put(ROW1); - put.addColumn(FAMILYA, QUAL, now, QUAL); - put.addColumn(FAMILYA, QUAL, now + 1, QUAL); - put.addColumn(FAMILYA, QUAL, now + 2, QUAL); - exportTable.put(put); - - put = new Put(ROW2); - put.addColumn(FAMILYA, QUAL, now, QUAL); - put.addColumn(FAMILYA, QUAL, now + 1, QUAL); - put.addColumn(FAMILYA, QUAL, now + 2, QUAL); - exportTable.put(put); - - // Run the export - String[] args = new String[] { exportTableName, FQ_OUTPUT_DIR, "1000" }; - assertTrue(runExport(args)); - - // Create the table for import - String importTableName = name.getMethodName() + "import1"; - Table importTable = UTIL.createTable(TableName.valueOf(importTableName), FAMILYA, 3); - - // Register the wal listener for the import table - RegionInfo region = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer() - .getRegions(importTable.getName()).get(0).getRegionInfo(); - TableWALActionListener walListener = new TableWALActionListener(region); - WAL wal = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(region); - wal.registerWALActionsListener(walListener); - - // Run the import with SKIP_WAL - args = new String[] { "-D" + Import.WAL_DURABILITY + "=" + Durability.SKIP_WAL.name(), - importTableName, FQ_OUTPUT_DIR }; - assertTrue(runImport(args)); - // Assert that the wal is not visisted - assertTrue(!walListener.isWALVisited()); - // Ensure that the count is 2 (only one version of key value is obtained) - assertTrue(getCount(importTable, null) == 2); - - // Run the import with the default durability option - importTableName = name.getMethodName() + "import2"; - importTable = UTIL.createTable(TableName.valueOf(importTableName), FAMILYA, 3); - region = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer() - .getRegions(importTable.getName()).get(0).getRegionInfo(); - wal = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(region); - walListener = new TableWALActionListener(region); - wal.registerWALActionsListener(walListener); - args = new String[] { importTableName, FQ_OUTPUT_DIR }; - assertTrue(runImport(args)); - // Assert that the wal is visisted - assertTrue(walListener.isWALVisited()); - // Ensure that the count is 2 (only one version of key value is obtained) - assertTrue(getCount(importTable, null) == 2); - } - } - - /** - * This listens to the {@link #visitLogEntryBeforeWrite(RegionInfo, WALKey, WALEdit)} to identify - * that an entry is written to the Write Ahead Log for the given table. - */ - private static class TableWALActionListener implements WALActionsListener { - - private RegionInfo regionInfo; - private boolean isVisited = false; - - public TableWALActionListener(RegionInfo region) { - this.regionInfo = region; - } - - @Override - public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) { - if ( - logKey.getTableName().getNameAsString() - .equalsIgnoreCase(this.regionInfo.getTable().getNameAsString()) && (!logEdit.isMetaEdit()) - ) { - isVisited = true; - } - } - - public boolean isWALVisited() { - return isVisited; - } - } - - /** - * Add cell tags to delete mutations, run export and import tool and verify that tags are present - * in import table also. - * @throws Throwable throws Throwable. - */ - @Test - public void testTagsAddition() throws Throwable { - final TableName exportTable = TableName.valueOf(name.getMethodName()); - TableDescriptor desc = TableDescriptorBuilder.newBuilder(exportTable) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5) - .setKeepDeletedCells(KeepDeletedCells.TRUE).build()) - .setCoprocessor(MetadataController.class.getName()).build(); - UTIL.getAdmin().createTable(desc); - - Table exportT = UTIL.getConnection().getTable(exportTable); - - // Add first version of QUAL - Put p = new Put(ROW1); - p.addColumn(FAMILYA, QUAL, now, QUAL); - exportT.put(p); - - // Add Delete family marker - Delete d = new Delete(ROW1, now + 3); - // Add test attribute to delete mutation. - d.setAttribute(TEST_ATTR, Bytes.toBytes(TEST_TAG)); - exportT.delete(d); - - // Run export tool with KeyValueCodecWithTags as Codec. This will ensure that export tool - // will use KeyValueCodecWithTags. - String[] args = new String[] { "-D" + ExportUtils.RAW_SCAN + "=true", - // This will make sure that codec will encode and decode tags in rpc call. - "-Dhbase.client.rpc.codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags", - exportTable.getNameAsString(), FQ_OUTPUT_DIR, "1000", // max number of key versions per key to - // export - }; - assertTrue(runExport(args)); - // Assert tag exists in exportTable - checkWhetherTagExists(exportTable, true); - - // Create an import table with MetadataController. - final TableName importTable = TableName.valueOf("importWithTestTagsAddition"); - TableDescriptor importTableDesc = TableDescriptorBuilder.newBuilder(importTable) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5) - .setKeepDeletedCells(KeepDeletedCells.TRUE).build()) - .setCoprocessor(MetadataController.class.getName()).build(); - UTIL.getAdmin().createTable(importTableDesc); - - // Run import tool. - args = new String[] { - // This will make sure that codec will encode and decode tags in rpc call. - "-Dhbase.client.rpc.codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags", - importTable.getNameAsString(), FQ_OUTPUT_DIR }; - assertTrue(runImport(args)); - // Make sure that tags exists in imported table. - checkWhetherTagExists(importTable, true); - } - - private void checkWhetherTagExists(TableName table, boolean tagExists) throws IOException { - List values = new ArrayList<>(); - for (HRegion region : UTIL.getHBaseCluster().getRegions(table)) { - Scan scan = new Scan(); - // Make sure to set rawScan to true so that we will get Delete Markers. - scan.setRaw(true); - scan.readAllVersions(); - scan.withStartRow(ROW1); - // Need to use RegionScanner instead of table#getScanner since the latter will - // not return tags since it will go through rpc layer and remove tags intentionally. - RegionScanner scanner = region.getScanner(scan); - scanner.next(values); - if (!values.isEmpty()) { - break; - } - } - boolean deleteFound = false; - for (Cell cell : values) { - if (PrivateCellUtil.isDelete(cell.getType().getCode())) { - deleteFound = true; - List tags = PrivateCellUtil.getTags(cell); - // If tagExists flag is true then validate whether tag contents are as expected. - if (tagExists) { - Assert.assertEquals(1, tags.size()); - for (Tag tag : tags) { - Assert.assertEquals(TEST_TAG, Tag.getValueAsString(tag)); - } - } else { - // If tagExists flag is disabled then check for 0 size tags. - assertEquals(0, tags.size()); - } - } - } - Assert.assertTrue(deleteFound); - } - - /* - * This co-proc will add a cell tag to delete mutation. - */ - public static class MetadataController implements RegionCoprocessor, RegionObserver { - @Override - public Optional getRegionObserver() { - return Optional.of(this); - } - - @Override - public void preBatchMutate(ObserverContext c, - MiniBatchOperationInProgress miniBatchOp) throws IOException { - if (c.getEnvironment().getRegion().getRegionInfo().getTable().isSystemTable()) { - return; - } - for (int i = 0; i < miniBatchOp.size(); i++) { - Mutation m = miniBatchOp.getOperation(i); - if (!(m instanceof Delete)) { - continue; - } - byte[] sourceOpAttr = m.getAttribute(TEST_ATTR); - if (sourceOpAttr == null) { - continue; - } - Tag sourceOpTag = new ArrayBackedTag(TEST_TAG_TYPE, sourceOpAttr); - List updatedCells = new ArrayList<>(); - for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { - Cell cell = cellScanner.current(); - List tags = PrivateCellUtil.getTags(cell); - tags.add(sourceOpTag); - Cell updatedCell = PrivateCellUtil.createCell(cell, tags); - updatedCells.add(updatedCell); - } - m.getFamilyCellMap().clear(); - // Clear and add new Cells to the Mutation. - for (Cell cell : updatedCells) { - Delete d = (Delete) m; - d.add(cell); - } - } - } - } - - /** - * Set hbase.client.rpc.codec and hbase.client.default.rpc.codec both to empty string This means - * it will use no Codec. Make sure that we don't return Tags in response. - * @throws Exception Exception - */ - @Test - public void testTagsWithEmptyCodec() throws Exception { - TableName tableName = TableName.valueOf(name.getMethodName()); - TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5) - .setKeepDeletedCells(KeepDeletedCells.TRUE).build()) - .setCoprocessor(MetadataController.class.getName()).build(); - UTIL.getAdmin().createTable(tableDesc); - Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.set(RPC_CODEC_CONF_KEY, ""); - conf.set(DEFAULT_CODEC_CLASS, ""); - try (Connection connection = ConnectionFactory.createConnection(conf); - Table table = connection.getTable(tableName)) { - // Add first version of QUAL - Put p = new Put(ROW1); - p.addColumn(FAMILYA, QUAL, now, QUAL); - table.put(p); - - // Add Delete family marker - Delete d = new Delete(ROW1, now + 3); - // Add test attribute to delete mutation. - d.setAttribute(TEST_ATTR, Bytes.toBytes(TEST_TAG)); - table.delete(d); - - // Since RPC_CODEC_CONF_KEY and DEFAULT_CODEC_CLASS is set to empty, it will use - // empty Codec and it shouldn't encode/decode tags. - Scan scan = new Scan().withStartRow(ROW1).setRaw(true); - ResultScanner scanner = table.getScanner(scan); - int count = 0; - Result result; - while ((result = scanner.next()) != null) { - List cells = result.listCells(); - assertEquals(2, cells.size()); - Cell cell = cells.get(0); - assertTrue(CellUtil.isDelete(cell)); - List tags = PrivateCellUtil.getTags(cell); - assertEquals(0, tags.size()); - count++; - } - assertEquals(1, count); - } finally { - UTIL.deleteTable(tableName); - } - } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExportBase.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExportBase.java new file mode 100644 index 000000000000..a91c2e36dcb0 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExportBase.java @@ -0,0 +1,980 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.apache.hadoop.hbase.HConstants.RPC_CODEC_CONF_KEY; +import static org.apache.hadoop.hbase.ipc.RpcClient.DEFAULT_CODEC_CLASS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ArrayBackedTag; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.RegionObserver; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.Import.KeyValueImporter; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.LauncherSecurityManager; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.mapreduce.Mapper.Context; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.ToolRunner; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for testing Import/Export. Shared logic without @BeforeAll/@AfterAll to allow + * subclasses to manage their own lifecycle. + */ +public class TestImportExportBase { + + private static final Logger LOG = LoggerFactory.getLogger(TestImportExportBase.class); + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final byte[] ROW1 = Bytes.toBytesBinary("\\x32row1"); + private static final byte[] ROW2 = Bytes.toBytesBinary("\\x32row2"); + private static final byte[] ROW3 = Bytes.toBytesBinary("\\x32row3"); + private static final String FAMILYA_STRING = "a"; + private static final String FAMILYB_STRING = "b"; + private static final byte[] FAMILYA = Bytes.toBytes(FAMILYA_STRING); + private static final byte[] FAMILYB = Bytes.toBytes(FAMILYB_STRING); + private static final byte[] QUAL = Bytes.toBytes("q"); + private static final String OUTPUT_DIR = "outputdir"; + private static String FQ_OUTPUT_DIR; + private static final String EXPORT_BATCH_SIZE = "100"; + + private static final long now = EnvironmentEdgeManager.currentTime(); + private final TableName EXPORT_TABLE = TableName.valueOf("export_table"); + private final TableName IMPORT_TABLE = TableName.valueOf("import_table"); + public static final byte TEST_TAG_TYPE = (byte) (Tag.CUSTOM_TAG_TYPE_RANGE + 1); + public static final String TEST_ATTR = "source_op"; + public static final String TEST_TAG = "test_tag"; + + protected String name; + + @BeforeEach + public void announce(TestInfo testInfo) { + name = testInfo.getTestMethod().get().getName(); + LOG.info("Running {}", name); + } + + @AfterEach + public void cleanup() throws Throwable { + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + fs.delete(new Path(OUTPUT_DIR), true); + if (UTIL.getAdmin().tableExists(EXPORT_TABLE)) { + UTIL.deleteTable(EXPORT_TABLE); + } + if (UTIL.getAdmin().tableExists(IMPORT_TABLE)) { + UTIL.deleteTable(IMPORT_TABLE); + } + } + + protected static void setUpBeforeClass() throws Exception { + // Up the handlers; this test needs more than usual. + UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); + UTIL.startMiniCluster(); + FQ_OUTPUT_DIR = + new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString(); + } + + /** + * Runs an export job with the specified command line args + * @return true if job completed successfully + */ + protected boolean runExport(String[] args) throws Throwable { + // need to make a copy of the configuration because to make sure different temp dirs are used. + int status = ToolRunner.run(new Configuration(UTIL.getConfiguration()), new Export(), args); + return status == 0; + } + + protected void runExportMain(String[] args) throws Throwable { + Export.main(args); + } + + /** + * Runs an import job with the specified command line args + * @return true if job completed successfully + */ + boolean runImport(String[] args) throws Throwable { + // need to make a copy of the configuration because to make sure different temp dirs are used. + int status = ToolRunner.run(new Configuration(UTIL.getConfiguration()), new Import(), args); + return status == 0; + } + + /** + * Test simple replication case with column mapping + */ + @Test + public void testSimpleCase() throws Throwable { + try (Table t = UTIL.createTable(TableName.valueOf(name), FAMILYA, 3);) { + Put p = new Put(ROW1); + p.addColumn(FAMILYA, QUAL, now, QUAL); + p.addColumn(FAMILYA, QUAL, now + 1, QUAL); + p.addColumn(FAMILYA, QUAL, now + 2, QUAL); + t.put(p); + p = new Put(ROW2); + p.addColumn(FAMILYA, QUAL, now, QUAL); + p.addColumn(FAMILYA, QUAL, now + 1, QUAL); + p.addColumn(FAMILYA, QUAL, now + 2, QUAL); + t.put(p); + p = new Put(ROW3); + p.addColumn(FAMILYA, QUAL, now, QUAL); + p.addColumn(FAMILYA, QUAL, now + 1, QUAL); + p.addColumn(FAMILYA, QUAL, now + 2, QUAL); + t.put(p); + } + + String[] args = new String[] { + // Only export row1 & row2. + "-D" + TableInputFormat.SCAN_ROW_START + "=\\x32row1", + "-D" + TableInputFormat.SCAN_ROW_STOP + "=\\x32row3", name, FQ_OUTPUT_DIR, "1000", // max + // number + // of key + // versions + // per key + // to + // export + }; + assertTrue(runExport(args)); + + final String IMPORT_TABLE = name + "import"; + try (Table t = UTIL.createTable(TableName.valueOf(IMPORT_TABLE), FAMILYB, 3);) { + args = + new String[] { "-D" + Import.CF_RENAME_PROP + "=" + FAMILYA_STRING + ":" + FAMILYB_STRING, + IMPORT_TABLE, FQ_OUTPUT_DIR }; + assertTrue(runImport(args)); + + Get g = new Get(ROW1); + g.setMaxVersions(); + Result r = t.get(g); + assertEquals(3, r.size()); + g = new Get(ROW2); + g.setMaxVersions(); + r = t.get(g); + assertEquals(3, r.size()); + g = new Get(ROW3); + r = t.get(g); + assertEquals(0, r.size()); + } + } + + /** + * Test export hbase:meta table + */ + @Test + public void testMetaExport() throws Throwable { + String[] args = + new String[] { TableName.META_TABLE_NAME.getNameAsString(), FQ_OUTPUT_DIR, "1", "0", "0" }; + assertTrue(runExport(args)); + } + + /** + * Test import data from 0.94 exported file + */ + @Test + public void testImport94Table() throws Throwable { + final String name = "exportedTableIn94Format"; + URL url = TestImportExport.class.getResource(name); + File f = new File(url.toURI()); + if (!f.exists()) { + LOG.warn("FAILED TO FIND " + f + "; skipping out on test"); + return; + } + assertTrue(f.exists()); + LOG.info("FILE=" + f); + Path importPath = new Path(f.toURI()); + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + fs.copyFromLocalFile(importPath, new Path(FQ_OUTPUT_DIR + Path.SEPARATOR + name)); + String IMPORT_TABLE = name; + try (Table t = UTIL.createTable(TableName.valueOf(IMPORT_TABLE), Bytes.toBytes("f1"), 3);) { + String[] args = new String[] { "-Dhbase.import.version=0.94", IMPORT_TABLE, FQ_OUTPUT_DIR }; + assertTrue(runImport(args)); + // @formatter:off + // exportedTableIn94Format contains 5 rows + // ROW COLUMN+CELL + // r1 column=f1:c1, timestamp=1383766761171, value=val1 + // r2 column=f1:c1, timestamp=1383766771642, value=val2 + // r3 column=f1:c1, timestamp=1383766777615, value=val3 + // r4 column=f1:c1, timestamp=1383766785146, value=val4 + // r5 column=f1:c1, timestamp=1383766791506, value=val5 + // @formatter:on + assertEquals(5, UTIL.countRows(t)); + } + } + + /** + * Test export scanner batching + */ + @Test + public void testExportScannerBatching() throws Throwable { + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(1).build()) + .build(); + UTIL.getAdmin().createTable(desc); + try (Table t = UTIL.getConnection().getTable(desc.getTableName());) { + + Put p = new Put(ROW1); + p.addColumn(FAMILYA, QUAL, now, QUAL); + p.addColumn(FAMILYA, QUAL, now + 1, QUAL); + p.addColumn(FAMILYA, QUAL, now + 2, QUAL); + p.addColumn(FAMILYA, QUAL, now + 3, QUAL); + p.addColumn(FAMILYA, QUAL, now + 4, QUAL); + t.put(p); + // added scanner batching arg. + String[] args = new String[] { "-D" + ExportUtils.EXPORT_BATCHING + "=" + EXPORT_BATCH_SIZE, + name, FQ_OUTPUT_DIR }; + assertTrue(runExport(args)); + + FileSystem fs = FileSystem.get(UTIL.getConfiguration()); + fs.delete(new Path(FQ_OUTPUT_DIR), true); + } + } + + @Test + public void testWithDeletes() throws Throwable { + TableDescriptor desc = TableDescriptorBuilder + .newBuilder(TableName.valueOf(name)).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(FAMILYA).setMaxVersions(5).setKeepDeletedCells(KeepDeletedCells.TRUE).build()) + .build(); + UTIL.getAdmin().createTable(desc); + try (Table t = UTIL.getConnection().getTable(desc.getTableName());) { + + Put p = new Put(ROW1); + p.addColumn(FAMILYA, QUAL, now, QUAL); + p.addColumn(FAMILYA, QUAL, now + 1, QUAL); + p.addColumn(FAMILYA, QUAL, now + 2, QUAL); + p.addColumn(FAMILYA, QUAL, now + 3, QUAL); + p.addColumn(FAMILYA, QUAL, now + 4, QUAL); + t.put(p); + + Delete d = new Delete(ROW1, now + 3); + t.delete(d); + d = new Delete(ROW1); + d.addColumns(FAMILYA, QUAL, now + 2); + t.delete(d); + } + + String[] args = + new String[] { "-D" + ExportUtils.RAW_SCAN + "=true", name, FQ_OUTPUT_DIR, "1000", // max + // number + // of key + // versions + // per key + // to + // export + }; + assertTrue(runExport(args)); + + final String IMPORT_TABLE = name + "import"; + desc = TableDescriptorBuilder + .newBuilder(TableName.valueOf(IMPORT_TABLE)).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(FAMILYA).setMaxVersions(5).setKeepDeletedCells(KeepDeletedCells.TRUE).build()) + .build(); + UTIL.getAdmin().createTable(desc); + try (Table t = UTIL.getConnection().getTable(desc.getTableName());) { + args = new String[] { IMPORT_TABLE, FQ_OUTPUT_DIR }; + assertTrue(runImport(args)); + + Scan s = new Scan(); + s.setMaxVersions(); + s.setRaw(true); + ResultScanner scanner = t.getScanner(s); + Result r = scanner.next(); + Cell[] res = r.rawCells(); + assertTrue(PrivateCellUtil.isDeleteFamily(res[0])); + assertEquals(now + 4, res[1].getTimestamp()); + assertEquals(now + 3, res[2].getTimestamp()); + assertTrue(CellUtil.isDelete(res[3])); + assertEquals(now + 2, res[4].getTimestamp()); + assertEquals(now + 1, res[5].getTimestamp()); + assertEquals(now, res[6].getTimestamp()); + } + } + + @Test + public void testWithMultipleDeleteFamilyMarkersOfSameRowSameFamily() throws Throwable { + final TableName exportTable = TableName.valueOf(name); + TableDescriptor desc = TableDescriptorBuilder + .newBuilder(TableName.valueOf(name)).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(FAMILYA).setMaxVersions(5).setKeepDeletedCells(KeepDeletedCells.TRUE).build()) + .build(); + UTIL.getAdmin().createTable(desc); + + Table exportT = UTIL.getConnection().getTable(exportTable); + + // Add first version of QUAL + Put p = new Put(ROW1); + p.addColumn(FAMILYA, QUAL, now, QUAL); + exportT.put(p); + + // Add Delete family marker + Delete d = new Delete(ROW1, now + 3); + exportT.delete(d); + + // Add second version of QUAL + p = new Put(ROW1); + p.addColumn(FAMILYA, QUAL, now + 5, "s".getBytes()); + exportT.put(p); + + // Add second Delete family marker + d = new Delete(ROW1, now + 7); + exportT.delete(d); + + String[] args = new String[] { "-D" + ExportUtils.RAW_SCAN + "=true", + exportTable.getNameAsString(), FQ_OUTPUT_DIR, "1000", // max number of key versions per key to + // export + }; + assertTrue(runExport(args)); + + final String importTable = name + "import"; + desc = TableDescriptorBuilder + .newBuilder(TableName.valueOf(importTable)).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(FAMILYA).setMaxVersions(5).setKeepDeletedCells(KeepDeletedCells.TRUE).build()) + .build(); + UTIL.getAdmin().createTable(desc); + + Table importT = UTIL.getConnection().getTable(TableName.valueOf(importTable)); + args = new String[] { importTable, FQ_OUTPUT_DIR }; + assertTrue(runImport(args)); + + Scan s = new Scan(); + s.setMaxVersions(); + s.setRaw(true); + + ResultScanner importedTScanner = importT.getScanner(s); + Result importedTResult = importedTScanner.next(); + + ResultScanner exportedTScanner = exportT.getScanner(s); + Result exportedTResult = exportedTScanner.next(); + try { + Result.compareResults(exportedTResult, importedTResult); + } catch (Throwable e) { + fail("Original and imported tables data comparision failed with error:" + e.getMessage()); + } finally { + exportT.close(); + importT.close(); + } + } + + /** + * Create a simple table, run an Export Job on it, Import with filtering on, verify counts, + * attempt with invalid values. + */ + @Test + public void testWithFilter() throws Throwable { + // Create simple table to export + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5).build()) + .build(); + UTIL.getAdmin().createTable(desc); + Table exportTable = UTIL.getConnection().getTable(desc.getTableName()); + + Put p1 = new Put(ROW1); + p1.addColumn(FAMILYA, QUAL, now, QUAL); + p1.addColumn(FAMILYA, QUAL, now + 1, QUAL); + p1.addColumn(FAMILYA, QUAL, now + 2, QUAL); + p1.addColumn(FAMILYA, QUAL, now + 3, QUAL); + p1.addColumn(FAMILYA, QUAL, now + 4, QUAL); + + // Having another row would actually test the filter. + Put p2 = new Put(ROW2); + p2.addColumn(FAMILYA, QUAL, now, QUAL); + + exportTable.put(Arrays.asList(p1, p2)); + + // Export the simple table + String[] args = new String[] { name, FQ_OUTPUT_DIR, "1000" }; + assertTrue(runExport(args)); + + // Import to a new table + final String IMPORT_TABLE = name + "import"; + desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(IMPORT_TABLE)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5).build()) + .build(); + UTIL.getAdmin().createTable(desc); + + Table importTable = UTIL.getConnection().getTable(desc.getTableName()); + args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + PrefixFilter.class.getName(), + "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1), IMPORT_TABLE, FQ_OUTPUT_DIR, + "1000" }; + assertTrue(runImport(args)); + + // get the count of the source table for that time range + PrefixFilter filter = new PrefixFilter(ROW1); + int count = getCount(exportTable, filter); + + assertEquals(count, getCount(importTable, null), + "Unexpected row count between export and import tables"); + + // and then test that a broken command doesn't bork everything - easier here because we don't + // need to re-run the export job + + args = new String[] { "-D" + Import.FILTER_CLASS_CONF_KEY + "=" + Filter.class.getName(), + "-D" + Import.FILTER_ARGS_CONF_KEY + "=" + Bytes.toString(ROW1) + "", name, FQ_OUTPUT_DIR, + "1000" }; + assertFalse(runImport(args)); + + // cleanup + exportTable.close(); + importTable.close(); + } + + /** + * Create a simple table, run an Export Job on it, Import with bulk output and enable largeResult + */ + @Test + public void testBulkImportAndLargeResult() throws Throwable { + // Create simple table to export + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5).build()) + .build(); + UTIL.getAdmin().createTable(desc); + Table exportTable = UTIL.getConnection().getTable(desc.getTableName()); + + Put p1 = new Put(ROW1); + p1.addColumn(FAMILYA, QUAL, now, QUAL); + + // Having another row would actually test the filter. + Put p2 = new Put(ROW2); + p2.addColumn(FAMILYA, QUAL, now, QUAL); + + exportTable.put(Arrays.asList(p1, p2)); + + // Export the simple table + String[] args = new String[] { name, FQ_OUTPUT_DIR, "1000" }; + assertTrue(runExport(args)); + + // Import to a new table + final String IMPORT_TABLE = name + "import"; + desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(IMPORT_TABLE)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5).build()) + .build(); + UTIL.getAdmin().createTable(desc); + + String O_OUTPUT_DIR = + new Path(OUTPUT_DIR + 1).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString(); + + args = new String[] { "-D" + Import.BULK_OUTPUT_CONF_KEY + "=" + O_OUTPUT_DIR, + "-D" + Import.HAS_LARGE_RESULT + "=" + true, IMPORT_TABLE, FQ_OUTPUT_DIR, "1000" }; + assertTrue(runImport(args)); + } + + /** + * Count the number of keyvalues in the specified table with the given filter + * @param table the table to scan + * @return the number of keyvalues found + */ + private int getCount(Table table, Filter filter) throws IOException { + Scan scan = new Scan(); + scan.setFilter(filter); + ResultScanner results = table.getScanner(scan); + int count = 0; + for (Result res : results) { + count += res.size(); + } + results.close(); + return count; + } + + /** + * test main method. Import should print help and call System.exit + */ + @Test + public void testImportMain() throws Throwable { + PrintStream oldPrintStream = System.err; + SecurityManager SECURITY_MANAGER = System.getSecurityManager(); + LauncherSecurityManager newSecurityManager = new LauncherSecurityManager(); + System.setSecurityManager(newSecurityManager); + ByteArrayOutputStream data = new ByteArrayOutputStream(); + String[] args = {}; + System.setErr(new PrintStream(data)); + try { + System.setErr(new PrintStream(data)); + Import.main(args); + fail("should be SecurityException"); + } catch (SecurityException e) { + assertEquals(-1, newSecurityManager.getExitCode()); + assertTrue(data.toString().contains("Wrong number of arguments:")); + assertTrue(data.toString().contains("-Dimport.bulk.output=/path/for/output")); + assertTrue(data.toString().contains("-Dimport.filter.class=")); + assertTrue(data.toString().contains("-Dimport.bulk.output=/path/for/output")); + assertTrue(data.toString().contains("-Dmapreduce.reduce.speculative=false")); + } finally { + System.setErr(oldPrintStream); + System.setSecurityManager(SECURITY_MANAGER); + } + } + + @Test + public void testExportScan() throws Exception { + int version = 100; + long startTime = EnvironmentEdgeManager.currentTime(); + long endTime = startTime + 1; + String prefix = "row"; + String label_0 = "label_0"; + String label_1 = "label_1"; + String[] args = { "table", "outputDir", String.valueOf(version), String.valueOf(startTime), + String.valueOf(endTime), prefix }; + Scan scan = ExportUtils.getScanFromCommandLine(UTIL.getConfiguration(), args); + assertEquals(version, scan.getMaxVersions()); + assertEquals(startTime, scan.getTimeRange().getMin()); + assertEquals(endTime, scan.getTimeRange().getMax()); + assertEquals(true, (scan.getFilter() instanceof PrefixFilter)); + assertEquals(0, + Bytes.compareTo(((PrefixFilter) scan.getFilter()).getPrefix(), Bytes.toBytesBinary(prefix))); + String[] argsWithLabels = + { "-D " + ExportUtils.EXPORT_VISIBILITY_LABELS + "=" + label_0 + "," + label_1, "table", + "outputDir", String.valueOf(version), String.valueOf(startTime), String.valueOf(endTime), + prefix }; + Configuration conf = new Configuration(UTIL.getConfiguration()); + // parse the "-D" options + String[] otherArgs = new GenericOptionsParser(conf, argsWithLabels).getRemainingArgs(); + Scan scanWithLabels = ExportUtils.getScanFromCommandLine(conf, otherArgs); + assertEquals(version, scanWithLabels.getMaxVersions()); + assertEquals(startTime, scanWithLabels.getTimeRange().getMin()); + assertEquals(endTime, scanWithLabels.getTimeRange().getMax()); + assertEquals(true, (scanWithLabels.getFilter() instanceof PrefixFilter)); + assertEquals(0, Bytes.compareTo(((PrefixFilter) scanWithLabels.getFilter()).getPrefix(), + Bytes.toBytesBinary(prefix))); + assertEquals(2, scanWithLabels.getAuthorizations().getLabels().size()); + assertEquals(label_0, scanWithLabels.getAuthorizations().getLabels().get(0)); + assertEquals(label_1, scanWithLabels.getAuthorizations().getLabels().get(1)); + } + + /** + * test main method. Export should print help and call System.exit + */ + @Test + public void testExportMain() throws Throwable { + PrintStream oldPrintStream = System.err; + SecurityManager SECURITY_MANAGER = System.getSecurityManager(); + LauncherSecurityManager newSecurityManager = new LauncherSecurityManager(); + System.setSecurityManager(newSecurityManager); + ByteArrayOutputStream data = new ByteArrayOutputStream(); + String[] args = {}; + System.setErr(new PrintStream(data)); + try { + System.setErr(new PrintStream(data)); + runExportMain(args); + fail("should be SecurityException"); + } catch (SecurityException e) { + assertEquals(-1, newSecurityManager.getExitCode()); + String errMsg = data.toString(); + assertTrue(errMsg.contains("Wrong number of arguments:")); + assertTrue( + errMsg.contains("Usage: Export [-D ]* [ " + + "[ []] [^[regex pattern] or [Prefix] to filter]]")); + assertTrue(errMsg.contains("-D hbase.mapreduce.scan.column.family=,, ...")); + assertTrue(errMsg.contains("-D hbase.mapreduce.include.deleted.rows=true")); + assertTrue(errMsg.contains("-D hbase.client.scanner.caching=100")); + assertTrue(errMsg.contains("-D hbase.export.scanner.batch=10")); + assertTrue(errMsg.contains("-D hbase.export.scanner.caching=100")); + } finally { + System.setErr(oldPrintStream); + System.setSecurityManager(SECURITY_MANAGER); + } + } + + /** + * Test map method of Importer + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + @Test + public void testKeyValueImporter() throws Throwable { + KeyValueImporter importer = new KeyValueImporter(); + Configuration configuration = new Configuration(); + Context ctx = mock(Context.class); + when(ctx.getConfiguration()).thenReturn(configuration); + + doAnswer(new Answer() { + + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + ImmutableBytesWritable writer = invocation.getArgument(0); + KeyValue key = invocation.getArgument(1); + assertEquals("Key", Bytes.toString(writer.get())); + assertEquals("row", Bytes.toString(CellUtil.cloneRow(key))); + return null; + } + }).when(ctx).write(any(), any()); + + importer.setup(ctx); + Result value = mock(Result.class); + KeyValue[] keys = { + new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"), + Bytes.toBytes("value")), + new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"), + Bytes.toBytes("value1")) }; + when(value.rawCells()).thenReturn(keys); + importer.map(new ImmutableBytesWritable(Bytes.toBytes("Key")), value, ctx); + + } + + /** + * Test addFilterAndArguments method of Import This method set couple parameters into + * Configuration + */ + @Test + public void testAddFilterAndArguments() throws IOException { + Configuration configuration = new Configuration(); + + List args = new ArrayList<>(); + args.add("param1"); + args.add("param2"); + + Import.addFilterAndArguments(configuration, FilterBase.class, args); + assertEquals("org.apache.hadoop.hbase.filter.FilterBase", + configuration.get(Import.FILTER_CLASS_CONF_KEY)); + assertEquals("param1,param2", configuration.get(Import.FILTER_ARGS_CONF_KEY)); + } + + @Test + public void testDurability() throws Throwable { + // Create an export table. + String exportTableName = name + "export"; + try (Table exportTable = UTIL.createTable(TableName.valueOf(exportTableName), FAMILYA, 3);) { + + // Insert some data + Put put = new Put(ROW1); + put.addColumn(FAMILYA, QUAL, now, QUAL); + put.addColumn(FAMILYA, QUAL, now + 1, QUAL); + put.addColumn(FAMILYA, QUAL, now + 2, QUAL); + exportTable.put(put); + + put = new Put(ROW2); + put.addColumn(FAMILYA, QUAL, now, QUAL); + put.addColumn(FAMILYA, QUAL, now + 1, QUAL); + put.addColumn(FAMILYA, QUAL, now + 2, QUAL); + exportTable.put(put); + + // Run the export + String[] args = new String[] { exportTableName, FQ_OUTPUT_DIR, "1000" }; + assertTrue(runExport(args)); + + // Create the table for import + String importTableName = name + "import1"; + Table importTable = UTIL.createTable(TableName.valueOf(importTableName), FAMILYA, 3); + + // Register the wal listener for the import table + RegionInfo region = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer() + .getRegions(importTable.getName()).get(0).getRegionInfo(); + TableWALActionListener walListener = new TableWALActionListener(region); + WAL wal = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(region); + wal.registerWALActionsListener(walListener); + + // Run the import with SKIP_WAL + args = new String[] { "-D" + Import.WAL_DURABILITY + "=" + Durability.SKIP_WAL.name(), + importTableName, FQ_OUTPUT_DIR }; + assertTrue(runImport(args)); + // Assert that the wal is not visisted + assertTrue(!walListener.isWALVisited()); + // Ensure that the count is 2 (only one version of key value is obtained) + assertTrue(getCount(importTable, null) == 2); + + // Run the import with the default durability option + importTableName = name + "import2"; + importTable = UTIL.createTable(TableName.valueOf(importTableName), FAMILYA, 3); + region = UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer() + .getRegions(importTable.getName()).get(0).getRegionInfo(); + wal = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(region); + walListener = new TableWALActionListener(region); + wal.registerWALActionsListener(walListener); + args = new String[] { importTableName, FQ_OUTPUT_DIR }; + assertTrue(runImport(args)); + // Assert that the wal is visisted + assertTrue(walListener.isWALVisited()); + // Ensure that the count is 2 (only one version of key value is obtained) + assertTrue(getCount(importTable, null) == 2); + } + } + + /** + * This listens to the {@link #visitLogEntryBeforeWrite(RegionInfo, WALKey, WALEdit)} to identify + * that an entry is written to the Write Ahead Log for the given table. + */ + private static class TableWALActionListener implements WALActionsListener { + + private RegionInfo regionInfo; + private boolean isVisited = false; + + public TableWALActionListener(RegionInfo region) { + this.regionInfo = region; + } + + @Override + public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) { + if ( + logKey.getTableName().getNameAsString() + .equalsIgnoreCase(this.regionInfo.getTable().getNameAsString()) && (!logEdit.isMetaEdit()) + ) { + isVisited = true; + } + } + + public boolean isWALVisited() { + return isVisited; + } + } + + /** + * Add cell tags to delete mutations, run export and import tool and verify that tags are present + * in import table also. + * @throws Throwable throws Throwable. + */ + @Test + public void testTagsAddition() throws Throwable { + final TableName exportTable = TableName.valueOf(name); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(exportTable) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5) + .setKeepDeletedCells(KeepDeletedCells.TRUE).build()) + .setCoprocessor(MetadataController.class.getName()).build(); + UTIL.getAdmin().createTable(desc); + + Table exportT = UTIL.getConnection().getTable(exportTable); + + // Add first version of QUAL + Put p = new Put(ROW1); + p.addColumn(FAMILYA, QUAL, now, QUAL); + exportT.put(p); + + // Add Delete family marker + Delete d = new Delete(ROW1, now + 3); + // Add test attribute to delete mutation. + d.setAttribute(TEST_ATTR, Bytes.toBytes(TEST_TAG)); + exportT.delete(d); + + // Run export tool with KeyValueCodecWithTags as Codec. This will ensure that export tool + // will use KeyValueCodecWithTags. + String[] args = new String[] { "-D" + ExportUtils.RAW_SCAN + "=true", + // This will make sure that codec will encode and decode tags in rpc call. + "-Dhbase.client.rpc.codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags", + exportTable.getNameAsString(), FQ_OUTPUT_DIR, "1000", // max number of key versions per key to + // export + }; + assertTrue(runExport(args)); + // Assert tag exists in exportTable + checkWhetherTagExists(exportTable, true); + + // Create an import table with MetadataController. + final TableName importTable = TableName.valueOf("importWithTestTagsAddition"); + TableDescriptor importTableDesc = TableDescriptorBuilder.newBuilder(importTable) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5) + .setKeepDeletedCells(KeepDeletedCells.TRUE).build()) + .setCoprocessor(MetadataController.class.getName()).build(); + UTIL.getAdmin().createTable(importTableDesc); + + // Run import tool. + args = new String[] { + // This will make sure that codec will encode and decode tags in rpc call. + "-Dhbase.client.rpc.codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags", + importTable.getNameAsString(), FQ_OUTPUT_DIR }; + assertTrue(runImport(args)); + // Make sure that tags exists in imported table. + checkWhetherTagExists(importTable, true); + } + + private void checkWhetherTagExists(TableName table, boolean tagExists) throws IOException { + List values = new ArrayList<>(); + for (HRegion region : UTIL.getHBaseCluster().getRegions(table)) { + Scan scan = new Scan(); + // Make sure to set rawScan to true so that we will get Delete Markers. + scan.setRaw(true); + scan.readAllVersions(); + scan.withStartRow(ROW1); + // Need to use RegionScanner instead of table#getScanner since the latter will + // not return tags since it will go through rpc layer and remove tags intentionally. + RegionScanner scanner = region.getScanner(scan); + scanner.next(values); + if (!values.isEmpty()) { + break; + } + } + boolean deleteFound = false; + for (Cell cell : values) { + if (PrivateCellUtil.isDelete(cell.getType().getCode())) { + deleteFound = true; + List tags = PrivateCellUtil.getTags(cell); + // If tagExists flag is true then validate whether tag contents are as expected. + if (tagExists) { + assertEquals(1, tags.size()); + for (Tag tag : tags) { + assertEquals(TEST_TAG, Tag.getValueAsString(tag)); + } + } else { + // If tagExists flag is disabled then check for 0 size tags. + assertEquals(0, tags.size()); + } + } + } + assertTrue(deleteFound); + } + + /* + * This co-proc will add a cell tag to delete mutation. + */ + public static class MetadataController implements RegionCoprocessor, RegionObserver { + @Override + public Optional getRegionObserver() { + return Optional.of(this); + } + + @Override + public void preBatchMutate(ObserverContext c, + MiniBatchOperationInProgress miniBatchOp) throws IOException { + if (c.getEnvironment().getRegion().getRegionInfo().getTable().isSystemTable()) { + return; + } + for (int i = 0; i < miniBatchOp.size(); i++) { + Mutation m = miniBatchOp.getOperation(i); + if (!(m instanceof Delete)) { + continue; + } + byte[] sourceOpAttr = m.getAttribute(TEST_ATTR); + if (sourceOpAttr == null) { + continue; + } + Tag sourceOpTag = new ArrayBackedTag(TEST_TAG_TYPE, sourceOpAttr); + List updatedCells = new ArrayList<>(); + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { + Cell cell = cellScanner.current(); + List tags = PrivateCellUtil.getTags(cell); + tags.add(sourceOpTag); + Cell updatedCell = PrivateCellUtil.createCell((ExtendedCell) cell, tags); + updatedCells.add(updatedCell); + } + m.getFamilyCellMap().clear(); + // Clear and add new Cells to the Mutation. + for (Cell cell : updatedCells) { + Delete d = (Delete) m; + d.add(cell); + } + } + } + } + + /** + * Set hbase.client.rpc.codec and hbase.client.default.rpc.codec both to empty string This means + * it will use no Codec. Make sure that we don't return Tags in response. + * @throws Exception Exception + */ + @Test + public void testTagsWithEmptyCodec() throws Exception { + TableName tableName = TableName.valueOf(name); + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA).setMaxVersions(5) + .setKeepDeletedCells(KeepDeletedCells.TRUE).build()) + .setCoprocessor(MetadataController.class.getName()).build(); + UTIL.getAdmin().createTable(tableDesc); + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.set(RPC_CODEC_CONF_KEY, ""); + conf.set(DEFAULT_CODEC_CLASS, ""); + try (Connection connection = ConnectionFactory.createConnection(conf); + Table table = connection.getTable(tableName)) { + // Add first version of QUAL + Put p = new Put(ROW1); + p.addColumn(FAMILYA, QUAL, now, QUAL); + table.put(p); + + // Add Delete family marker + Delete d = new Delete(ROW1, now + 3); + // Add test attribute to delete mutation. + d.setAttribute(TEST_ATTR, Bytes.toBytes(TEST_TAG)); + table.delete(d); + + // Since RPC_CODEC_CONF_KEY and DEFAULT_CODEC_CLASS is set to empty, it will use + // empty Codec and it shouldn't encode/decode tags. + Scan scan = new Scan().withStartRow(ROW1).setRaw(true); + ResultScanner scanner = table.getScanner(scan); + int count = 0; + Result result; + while ((result = scanner.next()) != null) { + List cells = result.listCells(); + assertEquals(2, cells.size()); + Cell cell = cells.get(0); + assertTrue(CellUtil.isDelete(cell)); + List tags = PrivateCellUtil.getTags(cell); + assertEquals(0, tags.size()); + count++; + } + assertEquals(1, count); + } finally { + UTIL.deleteTable(tableName); + } + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanOBBToOPP.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanOBBToOPP.java index 7d833eb66a1a..fe3d703a289b 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanOBBToOPP.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanOBBToOPP.java @@ -19,13 +19,13 @@ import java.io.IOException; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -@Category({ MapReduceTests.class, MediumTests.class }) +@Category({ MapReduceTests.class, LargeTests.class }) public class TestTableInputFormatScanOBBToOPP extends TestTableInputFormatScanBase { @ClassRule diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java index 3b7392b3ae45..5248c9dae59e 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java @@ -29,17 +29,9 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.security.HBaseKerberosUtils; -import org.apache.hadoop.hbase.security.access.AccessController; -import org.apache.hadoop.hbase.security.access.PermissionStorage; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; -import org.apache.hadoop.hbase.security.provider.SaslClientAuthenticationProviders; import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; -import org.apache.hadoop.hbase.security.token.TokenProvider; -import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.io.LongWritable; @@ -48,10 +40,8 @@ import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.junit.After; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -59,7 +49,7 @@ /** * Test different variants of initTableMapperJob method */ -@Category({ MapReduceTests.class, MediumTests.class }) +@Category({ MapReduceTests.class, LargeTests.class }) public class TestTableMapReduceUtil { private static final String HTTP_PRINCIPAL = "HTTP/localhost"; @@ -67,11 +57,6 @@ public class TestTableMapReduceUtil { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestTableMapReduceUtil.class); - @After - public void after() { - SaslClientAuthenticationProviders.reset(); - } - /* * initTableSnapshotMapperJob is tested in {@link TestTableSnapshotInputFormat} because the method * depends on an online cluster. @@ -134,33 +119,6 @@ public void testInitTableMapperJob4() throws Exception { assertEquals("Table", job.getConfiguration().get(TableInputFormat.INPUT_TABLE)); } - private static Closeable startSecureMiniCluster(HBaseTestingUtility util, MiniKdc kdc, - String principal) throws Exception { - Configuration conf = util.getConfiguration(); - - SecureTestUtil.enableSecurity(conf); - VisibilityTestUtil.enableVisiblityLabels(conf); - SecureTestUtil.verifyConfiguration(conf); - - conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, - AccessController.class.getName() + ',' + TokenProvider.class.getName()); - - HBaseKerberosUtils.setSecuredConfiguration(conf, principal + '@' + kdc.getRealm(), - HTTP_PRINCIPAL + '@' + kdc.getRealm()); - - KerberosName.resetDefaultRealm(); - - util.startMiniCluster(); - try { - util.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); - } catch (Exception e) { - util.shutdownMiniCluster(); - throw e; - } - - return util::shutdownMiniCluster; - } - @Test public void testInitCredentialsForCluster1() throws Exception { HBaseTestingUtility util1 = new HBaseTestingUtility(); @@ -199,8 +157,9 @@ public void testInitCredentialsForCluster2() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal); - Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { + try ( + Closeable util1Closeable = util1.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL); + Closeable util2Closeable = util2.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { Configuration conf1 = util1.getConfiguration(); Job job = Job.getInstance(conf1); @@ -233,7 +192,8 @@ public void testInitCredentialsForCluster3() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal)) { + try ( + Closeable util1Closeable = util1.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { HBaseTestingUtility util2 = new HBaseTestingUtility(); // Assume util2 is insecure cluster @@ -269,7 +229,8 @@ public void testInitCredentialsForCluster4() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { + try ( + Closeable util2Closeable = util2.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { Configuration conf1 = util1.getConfiguration(); Job job = Job.getInstance(conf1); diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java index 801099819b76..9170b26564de 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java @@ -29,6 +29,8 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.junit.After; @@ -127,4 +129,43 @@ public void testTableOutputFormatWhenWalIsOFFForDelete() Assert.assertEquals("Durability of the mutation should be SKIP_WAL", Durability.SKIP_WAL, delete.getDurability()); } + + @Test + public void testOutputCommitterConfiguration() throws IOException, InterruptedException { + // 1. Verify it returns the default committer when the property is not set. + conf.unset(TableOutputFormat.OUTPUT_COMMITTER_CLASS); + tableOutputFormat.setConf(conf); + Assert.assertEquals("Should use default committer", TableOutputCommitter.class, + tableOutputFormat.getOutputCommitter(context).getClass()); + + // 2. Verify it returns the custom committer when the property is set. + conf.set(TableOutputFormat.OUTPUT_COMMITTER_CLASS, DummyCommitter.class.getName()); + tableOutputFormat.setConf(conf); + Assert.assertEquals("Should use custom committer", DummyCommitter.class, + tableOutputFormat.getOutputCommitter(context).getClass()); + } + + // Simple dummy committer for testing + public static class DummyCommitter extends OutputCommitter { + @Override + public void setupJob(JobContext jobContext) { + } + + @Override + public void setupTask(TaskAttemptContext taskContext) { + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) { + return false; + } + + @Override + public void commitTask(TaskAttemptContext taskContext) { + } + + @Override + public void abortTask(TaskAttemptContext taskContext) { + } + } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java index eca275cf0a90..9909ce15af9c 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.TestTableSnapshotScanner; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit; +import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; @@ -583,4 +584,104 @@ public void testCleanRestoreDir() throws Exception { TableSnapshotInputFormat.cleanRestoreDir(job, snapshotName); Assert.assertFalse(fs.exists(restorePath)); } + + /** + * Test that explicitly restores a snapshot to a temp directory and reads the restored regions via + * ClientSideRegionScanner through a MapReduce job. + *

+ * This test verifies the full workflow: 1. Create and load a table with data 2. Create a snapshot + * and restore the snapshot to a temporary directory 3. Configure a job to read the restored + * regions via ClientSideRegionScanner using TableSnapshotInputFormat and verify that it succeeds + * 4. Delete restored temporary directory 5. Configure a new job and verify that it fails + */ + @Test + public void testReadFromRestoredSnapshotViaMR() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final String snapshotName = tableName + "_snapshot"; + try { + if (UTIL.getAdmin().tableExists(tableName)) { + UTIL.deleteTable(tableName); + } + UTIL.createTable(tableName, FAMILIES, new byte[][] { bbb, yyy }); + + Admin admin = UTIL.getAdmin(); + int regionNum = admin.getRegions(tableName).size(); + LOG.info("Created table with {} regions", regionNum); + + Table table = UTIL.getConnection().getTable(tableName); + UTIL.loadTable(table, FAMILIES); + table.close(); + + Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()); + FileSystem fs = rootDir.getFileSystem(UTIL.getConfiguration()); + SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName, Arrays.asList(FAMILIES), + null, snapshotName, rootDir, fs, true); + Path tempRestoreDir = UTIL.getDataTestDirOnTestFS("restore_" + snapshotName); + RestoreSnapshotHelper.copySnapshotForScanner(UTIL.getConfiguration(), fs, rootDir, + tempRestoreDir, snapshotName); + Assert.assertTrue("Restore directory should exist", fs.exists(tempRestoreDir)); + + Job job = Job.getInstance(UTIL.getConfiguration()); + job.setJarByClass(TestTableSnapshotInputFormat.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), + TestTableSnapshotInputFormat.class); + Scan scan = new Scan().withStartRow(getStartRow()).withStopRow(getEndRow()); + Configuration conf = job.getConfiguration(); + conf.set("hbase.TableSnapshotInputFormat.snapshot.name", snapshotName); + conf.set("hbase.TableSnapshotInputFormat.restore.dir", tempRestoreDir.toString()); + conf.setInt("hbase.mapreduce.splits.per.region", 1); + job.setReducerClass(TestTableSnapshotReducer.class); + job.setNumReduceTasks(1); + job.setOutputFormatClass(NullOutputFormat.class); + TableMapReduceUtil.initTableMapperJob(snapshotName, // table name (snapshot name in this case) + scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, + false, false, TableSnapshotInputFormat.class); + TableMapReduceUtil.resetCacheConfig(conf); + Assert.assertTrue(job.waitForCompletion(true)); + Assert.assertTrue(job.isSuccessful()); + + // Now verify that job fails when restore directory is deleted + Assert.assertTrue(fs.delete(tempRestoreDir, true)); + Assert.assertFalse("Restore directory should not exist after deletion", + fs.exists(tempRestoreDir)); + Job failureJob = Job.getInstance(UTIL.getConfiguration()); + failureJob.setJarByClass(TestTableSnapshotInputFormat.class); + TableMapReduceUtil.addDependencyJarsForClasses(failureJob.getConfiguration(), + TestTableSnapshotInputFormat.class); + Configuration failureConf = failureJob.getConfiguration(); + // Configure job to use the deleted restore directory + failureConf.set("hbase.TableSnapshotInputFormat.snapshot.name", snapshotName); + failureConf.set("hbase.TableSnapshotInputFormat.restore.dir", tempRestoreDir.toString()); + failureConf.setInt("hbase.mapreduce.splits.per.region", 1); + failureJob.setReducerClass(TestTableSnapshotReducer.class); + failureJob.setNumReduceTasks(1); + failureJob.setOutputFormatClass(NullOutputFormat.class); + + TableMapReduceUtil.initTableMapperJob(snapshotName, scan, TestTableSnapshotMapper.class, + ImmutableBytesWritable.class, NullWritable.class, failureJob, false, false, + TableSnapshotInputFormat.class); + TableMapReduceUtil.resetCacheConfig(failureConf); + + Assert.assertFalse("Restore directory should not exist before job execution", + fs.exists(tempRestoreDir)); + failureJob.waitForCompletion(true); + + Assert.assertFalse("Job should fail since the restored snapshot directory is deleted", + failureJob.isSuccessful()); + + } finally { + try { + if (UTIL.getAdmin().tableExists(tableName)) { + UTIL.deleteTable(tableName); + } + } catch (Exception e) { + LOG.warn("Error deleting table", e); + } + try { + UTIL.getAdmin().deleteSnapshot(snapshotName); + } catch (Exception e) { + LOG.warn("Error deleting snapshot", e); + } + } + } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java index 70602a371668..6fdfb2bb8e2d 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java @@ -21,24 +21,43 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; -@Category({ MapReduceTests.class, SmallTests.class }) +@Category({ MapReduceTests.class, MediumTests.class }) public class TestWALInputFormat { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestWALInputFormat.class); + @BeforeClass + public static void setupClass() throws Exception { + TEST_UTIL.startMiniCluster(); + TEST_UTIL.createWALRootDir(); + } + /** * Test the primitive start/end time filtering. */ @@ -74,4 +93,36 @@ public void testAddFile() { WALInputFormat.addFile(lfss, lfs, now, now); assertEquals(8, lfss.size()); } + + @Test + public void testHandlesArchivedWALFiles() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + JobContext ctx = Mockito.mock(JobContext.class); + Mockito.when(ctx.getConfiguration()).thenReturn(conf); + Job job = Job.getInstance(conf); + TableMapReduceUtil.initCredentialsForCluster(job, conf); + Mockito.when(ctx.getCredentials()).thenReturn(job.getCredentials()); + + // Setup WAL file, then archive it + HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(0); + AbstractFSWAL wal = (AbstractFSWAL) rs.getWALs().get(0); + Path walPath = wal.getCurrentFileName(); + TEST_UTIL.getConfiguration().set(FileInputFormat.INPUT_DIR, walPath.toString()); + TEST_UTIL.getConfiguration().set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";"); + + Path rootDir = CommonFSUtils.getWALRootDir(conf); + Path archiveWal = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + archiveWal = new Path(archiveWal, walPath.getName()); + TEST_UTIL.getTestFileSystem().delete(walPath, true); + TEST_UTIL.getTestFileSystem().mkdirs(archiveWal.getParent()); + TEST_UTIL.getTestFileSystem().create(archiveWal).close(); + + // Test for that we can read from the archived WAL file + WALInputFormat wif = new WALInputFormat(); + List splits = wif.getSplits(ctx); + assertEquals(1, splits.size()); + WALInputFormat.WALSplit split = (WALInputFormat.WALSplit) splits.get(0); + assertEquals(archiveWal.toString(), split.getLogFileName()); + } + } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java index c6e51eee40ff..fcfb149c2fd7 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -36,7 +36,10 @@ import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -45,11 +48,15 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ClientSideRegionScanner; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.WALPlayer.WALKeyValueMapper; import org.apache.hadoop.hbase.regionserver.TestRecoveredEdits; @@ -114,6 +121,50 @@ public static void afterClass() throws Exception { logFs.delete(walRootDir, true); } + @Test + public void testDiskBasedSortingEnabled() throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + final byte[] COLUMN2 = Bytes.toBytes("c2"); + final byte[] ROW = Bytes.toBytes("row"); + Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); + + // put a row into the first table + Put p = new Put(ROW); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + p.addColumn(FAMILY, COLUMN2, COLUMN2); + t1.put(p); + // delete one column + Delete d = new Delete(ROW); + d.addColumns(FAMILY, COLUMN1); + t1.delete(d); + + // replay the WAL, map table 1 to table 2 + WAL log = cluster.getRegionServer(0).getWAL(null); + log.rollWriter(); + String walInputDir = new Path(cluster.getMaster().getMasterFileSystem().getWALRootDir(), + HConstants.HREGION_LOGDIR_NAME).toString(); + + Configuration configuration = TEST_UTIL.getConfiguration(); + configuration.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); + WALPlayer player = new WALPlayer(configuration); + String optionName = "_test_.name"; + configuration.set(optionName, "1000"); + player.setupTime(configuration, optionName); + assertEquals(1000, configuration.getLong(optionName, 0)); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); + + // verify the WAL was player into table 2 + Get g = new Get(ROW); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + } + /** * Test that WALPlayer can replay recovered.edits files. */ @@ -123,19 +174,22 @@ public void testPlayingRecoveredEdit() throws Exception { TEST_UTIL.createTable(tn, TestRecoveredEdits.RECOVEREDEDITS_COLUMNFAMILY); // Copy testing recovered.edits file that is over under hbase-server test resources // up into a dir in our little hdfs cluster here. - String hbaseServerTestResourcesEdits = - System.getProperty("test.build.classes") + "/../../../hbase-server/src/test/resources/" - + TestRecoveredEdits.RECOVEREDEDITS_PATH.getName(); - assertTrue(new File(hbaseServerTestResourcesEdits).exists()); - FileSystem dfs = TEST_UTIL.getDFSCluster().getFileSystem(); - // Target dir. - Path targetDir = new Path("edits").makeQualified(dfs.getUri(), dfs.getHomeDirectory()); - assertTrue(dfs.mkdirs(targetDir)); - dfs.copyFromLocalFile(new Path(hbaseServerTestResourcesEdits), targetDir); - assertEquals(0, - ToolRunner.run(new WALPlayer(this.conf), new String[] { targetDir.toString() })); - // I don't know how many edits are in this file for this table... so just check more than 1. - assertTrue(TEST_UTIL.countRows(tn) > 0); + runWithDiskBasedSortingDisabledAndEnabled(() -> { + String hbaseServerTestResourcesEdits = + System.getProperty("test.build.classes") + "/../../../hbase-server/src/test/resources/" + + TestRecoveredEdits.RECOVEREDEDITS_PATH.getName(); + assertTrue(new File(hbaseServerTestResourcesEdits).exists()); + FileSystem dfs = TEST_UTIL.getDFSCluster().getFileSystem(); + // Target dir. + Path targetDir = new Path("edits").makeQualified(dfs.getUri(), dfs.getHomeDirectory()); + assertTrue(dfs.mkdirs(targetDir)); + dfs.copyFromLocalFile(new Path(hbaseServerTestResourcesEdits), targetDir); + assertEquals(0, + ToolRunner.run(new WALPlayer(this.conf), new String[] { targetDir.toString() })); + // I don't know how many edits are in this file for this table... so just check more than 1. + assertTrue(TEST_UTIL.countRows(tn) > 0); + dfs.delete(targetDir, true); + }); } /** @@ -150,7 +204,7 @@ public void testWALPlayerBulkLoadWithOverriddenTimestamps() throws Exception { final byte[] column1 = Bytes.toBytes("c1"); final byte[] column2 = Bytes.toBytes("c2"); final byte[] row = Bytes.toBytes("row"); - Table table = TEST_UTIL.createTable(tableName, family); + final Table table = TEST_UTIL.createTable(tableName, family); long now = EnvironmentEdgeManager.currentTime(); // put a row into the first table @@ -187,29 +241,145 @@ public void testWALPlayerBulkLoadWithOverriddenTimestamps() throws Exception { configuration.set(WALPlayer.BULK_OUTPUT_CONF_KEY, outPath); configuration.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); - WALPlayer player = new WALPlayer(configuration); - assertEquals(0, ToolRunner.run(configuration, player, - new String[] { walInputDir, tableName.getNameAsString() })); + final byte[] finalLastVal = lastVal; - Get g = new Get(row); - Result result = table.get(g); - byte[] value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); - assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(lastVal))); + runWithDiskBasedSortingDisabledAndEnabled(() -> { + WALPlayer player = new WALPlayer(configuration); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName.getNameAsString() })); - table = TEST_UTIL.truncateTable(tableName); - g = new Get(row); - result = table.get(g); - assertThat(result.listCells(), nullValue()); + Get g = new Get(row); + Result result = table.get(g); + byte[] value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); + assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(finalLastVal))); - BulkLoadHFiles.create(configuration).bulkLoad(tableName, - new Path(outPath, tableName.getNameAsString())); + TEST_UTIL.truncateTable(tableName); + g = new Get(row); + result = table.get(g); + assertThat(result.listCells(), nullValue()); - g = new Get(row); - result = table.get(g); - value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); + BulkLoadHFiles.create(configuration).bulkLoad(tableName, + new Path(outPath, tableName.getNameAsString())); - assertThat(result.listCells(), notNullValue()); - assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(lastVal))); + g = new Get(row); + result = table.get(g); + value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); + + assertThat(result.listCells(), notNullValue()); + assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(finalLastVal))); + + // cleanup + Path out = new Path(outPath); + FileSystem fs = out.getFileSystem(configuration); + assertTrue(fs.delete(out, true)); + }); + } + + /** + * Tests that the sequence IDs of cells are retained in the resulting HFile and usable by a + * RegionScanner. It does this by running the WALPlayer multiple times and using a RegionScanner + * to read the files; without sequence IDs, the files will be sorted by size or name and will not + * always return the correct result. + */ + @Test + public void testMaxSeqIdHFileMetadata() throws Exception { + final int numEdits = 20; + final int flushInterval = 10; + + // Phase 1: Setup test data and configuration + final TableName tableName = TableName.valueOf(name.getMethodName()); + final byte[] family = Bytes.toBytes("family"); + final byte[] column = Bytes.toBytes("c1"); + final byte[] row = Bytes.toBytes("row"); + final Table table = TEST_UTIL.createTable(tableName, family); + + long now = EnvironmentEdgeManager.currentTime(); + { + Put put = new Put(row); + put.addColumn(family, column, now, column); + table.put(put); + } + + String walInputDir = new Path(cluster.getMaster().getMasterFileSystem().getWALRootDir(), + HConstants.HREGION_LOGDIR_NAME).toString(); + String walPlayerOutputRoot = "/tmp/" + name.getMethodName(); + + Configuration walPlayerConfig = new Configuration(TEST_UTIL.getConfiguration()); + walPlayerConfig.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); + walPlayerConfig.setBoolean(HFileOutputFormat2.SET_MAX_SEQ_ID_KEY, true); + + // Phase 2: Write edits with periodic WAL rolling and WALPlayer execution + int walPlayerRunCount = 0; + byte[] lastVal = null; + + for (int i = 0; i < numEdits; i++) { + lastVal = new byte[12]; + ThreadLocalRandom.current().nextBytes(lastVal); + + Put put = new Put(row); + put.addColumn(family, column, now, lastVal); + table.put(put); + + // Roll WALs and run WALPlayer every flushInterval iterations + if (i > 0 && (i % flushInterval == 0) || i + 1 == numEdits) { + WAL log = cluster.getRegionServer(0).getWAL(null); + log.rollWriter(); + + walPlayerRunCount++; + String walPlayerRunDir = walPlayerOutputRoot + "/run_" + walPlayerRunCount; + Configuration runConfig = new Configuration(walPlayerConfig); + runConfig.set(WALPlayer.BULK_OUTPUT_CONF_KEY, walPlayerRunDir); + + WALPlayer player = new WALPlayer(runConfig); + assertEquals(0, ToolRunner.run(runConfig, player, + new String[] { walInputDir, tableName.getNameAsString() })); + } + } + + table.close(); + + final byte[] finalLastVal = lastVal; + + // Phase 3: Collect all generated HFiles into proper structure for region scanner + TableDescriptor htd = TEST_UTIL.getAdmin().getDescriptor(tableName); + RegionInfo regionInfo = cluster.getRegions(tableName).get(0).getRegionInfo(); + FileSystem fs = cluster.getRegionServer(0).getFileSystem(); + + Path regionOutPath = CommonFSUtils.getRegionDir(new Path(walPlayerOutputRoot), + htd.getTableName(), regionInfo.getEncodedName()); + Path familyOutPath = new Path(regionOutPath, new String(family)); + fs.mkdirs(familyOutPath); + + // Copy all HFiles from each WALPlayer run + for (int i = 1; i <= walPlayerRunCount; i++) { + Path walPlayerRunPath = new Path(walPlayerOutputRoot, "run_" + i); + RemoteIterator files = + fs.listFiles(new Path(walPlayerRunPath, tableName.getNamespaceAsString()), true); + + while (files.hasNext()) { + LocatedFileStatus fileStatus = files.next(); + // Skip hidden/metadata files (starting with '.') + if (fileStatus.isFile() && !fileStatus.getPath().getName().startsWith(".")) { + FileUtil.copy(fs, fileStatus.getPath(), fs, + new Path(familyOutPath, fileStatus.getPath().getName()), false, walPlayerConfig); + } + } + } + + // Phase 4: Verify sequence IDs are preserved correctly + Scan scan = new Scan(); + try (ClientSideRegionScanner scanner = new ClientSideRegionScanner(walPlayerConfig, fs, + new Path(walPlayerOutputRoot), htd, regionInfo, scan, null)) { + + // Verify exactly one row returned + Result result = scanner.next(); + assertThat(result, notNullValue()); + assertThat(result.listCells(), notNullValue()); + + // Verify the value with highest sequence ID (from last iteration) wins + byte[] value = CellUtil.cloneValue(result.getColumnLatestCell(family, column)); + assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(finalLastVal))); + } } /** @@ -244,18 +414,21 @@ public void testWALPlayer() throws Exception { Configuration configuration = TEST_UTIL.getConfiguration(); WALPlayer player = new WALPlayer(configuration); - String optionName = "_test_.name"; - configuration.set(optionName, "1000"); - player.setupTime(configuration, optionName); - assertEquals(1000, configuration.getLong(optionName, 0)); - assertEquals(0, ToolRunner.run(configuration, player, - new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); - // verify the WAL was player into table 2 - Get g = new Get(ROW); - Result r = t2.get(g); - assertEquals(1, r.size()); - assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + runWithDiskBasedSortingDisabledAndEnabled(() -> { + String optionName = "_test_.name"; + configuration.set(optionName, "1000"); + player.setupTime(configuration, optionName); + assertEquals(1000, configuration.getLong(optionName, 0)); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); + + // verify the WAL was player into table 2 + Get g = new Get(ROW); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + }); } /** @@ -271,6 +444,55 @@ public void testWALKeyValueMapperWithDeprecatedConfig() throws Exception { testWALKeyValueMapper("hlog.input.tables"); } + @Test + public void testInsertionOrderPreserved() throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[] COLUMN = Bytes.toBytes("c1"); + final byte[] ROW = Bytes.toBytes("row"); + Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); + + // put a row into the first table + Put p = new Put(ROW); + p.addColumn(FAMILY, COLUMN, Bytes.toBytes("aaa")); + p.addColumn(FAMILY, COLUMN, Bytes.toBytes("zzz")); + t1.put(p); + + // replay the WAL, map table 1 to table 2 + WAL log = cluster.getRegionServer(0).getWAL(null); + log.rollWriter(); + String walInputDir = new Path(cluster.getMaster().getMasterFileSystem().getWALRootDir(), + HConstants.HREGION_LOGDIR_NAME).toString(); + + Configuration configuration = TEST_UTIL.getConfiguration(); + WALPlayer player = new WALPlayer(configuration); + + configuration.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); + + try { + String optionName = "_test_.name"; + configuration.set(optionName, "1000"); + player.setupTime(configuration, optionName); + String outPath = "/tmp/" + name.getMethodName(); + configuration.set(WALPlayer.BULK_OUTPUT_CONF_KEY, outPath); + assertEquals(1000, configuration.getLong(optionName, 0)); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); + + BulkLoadHFiles.create(configuration).bulkLoad(tableName2, new Path(outPath)); + + Get g = new Get(ROW); + Result r = t2.get(g); + Cell cell = r.getColumnLatestCell(FAMILY, COLUMN); + String value = Bytes.toString(CellUtil.cloneValue(cell)); + assertEquals("zzz", value); + } finally { + configuration.unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); + } + } + private void testWALKeyValueMapper(final String tableConfigKey) throws Exception { Configuration configuration = new Configuration(); configuration.set(tableConfigKey, "table"); @@ -335,7 +557,29 @@ public void testMainMethod() throws Exception { System.setErr(oldPrintStream); System.setSecurityManager(SECURITY_MANAGER); } + } + + private static void runWithDiskBasedSortingDisabledAndEnabled(TestMethod method) + throws Exception { + TEST_UTIL.getConfiguration().setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + false); + try { + method.run(); + } finally { + TEST_UTIL.getConfiguration().unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); + } + + TEST_UTIL.getConfiguration().setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + true); + try { + method.run(); + } finally { + TEST_UTIL.getConfiguration().unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); + } + } + private interface TestMethod { + void run() throws Exception; } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ConsecutiveExportsTestBase.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ConsecutiveExportsTestBase.java new file mode 100644 index 000000000000..852602e80c94 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ConsecutiveExportsTestBase.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.TestTemplate; + +public class ConsecutiveExportsTestBase extends ExportSnapshotTestBase { + + protected ConsecutiveExportsTestBase(boolean mob) { + super(mob); + } + + @TestTemplate + public void testConsecutiveExports() throws Exception { + Path copyDir = getLocalDestinationDir(TEST_UTIL); + testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); + testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); + removeExportDir(copyDir); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportFileSystemStateTestBase.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportFileSystemStateTestBase.java new file mode 100644 index 000000000000..6ddfc211e75d --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportFileSystemStateTestBase.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.junit.jupiter.api.TestTemplate; + +public class ExportFileSystemStateTestBase extends ExportSnapshotTestBase { + + protected ExportFileSystemStateTestBase(boolean mob) { + super(mob); + } + + /** + * Verify if exported snapshot and copied files matches the original one. + */ + @TestTemplate + public void testExportFileSystemState() throws Exception { + testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); + } + + @TestTemplate + public void testExportFileSystemStateWithSkipTmp() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); + try { + testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); + } finally { + TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); + } + } + + @TestTemplate + public void testEmptyExportFileSystemState() throws Exception { + testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportFileSystemStateWithMergeOrSplitRegionTestBase.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportFileSystemStateWithMergeOrSplitRegionTestBase.java new file mode 100644 index 000000000000..1405e338e560 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportFileSystemStateWithMergeOrSplitRegionTestBase.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HFileTestUtil; +import org.junit.jupiter.api.TestTemplate; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +public class ExportFileSystemStateWithMergeOrSplitRegionTestBase extends ExportSnapshotTestBase { + + protected ExportFileSystemStateWithMergeOrSplitRegionTestBase(boolean mob) { + super(mob); + } + + @TestTemplate + public void testExportFileSystemStateWithMergeRegion() throws Exception { + // disable compaction + admin.compactionSwitch(false, + admin.getRegionServers().stream().map(a -> a.getServerName()).collect(Collectors.toList())); + // create Table + String suffix = mob ? methodName + "-mob" : methodName; + TableName tableName0 = TableName.valueOf("testtb-" + suffix + "-1"); + String snapshotName0 = "snaptb0-" + suffix + "-1"; + admin.createTable( + TableDescriptorBuilder.newBuilder(tableName0) + .setColumnFamilies( + Lists.newArrayList(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build())) + .build(), + new byte[][] { Bytes.toBytes("2") }); + // put some data + try (Table table = admin.getConnection().getTable(tableName0)) { + table.put(new Put(Bytes.toBytes("1")).addColumn(FAMILY, null, Bytes.toBytes("1"))); + table.put(new Put(Bytes.toBytes("2")).addColumn(FAMILY, null, Bytes.toBytes("2"))); + } + List regions = admin.getRegions(tableName0); + assertEquals(2, regions.size()); + tableNumFiles = regions.size(); + // merge region + admin.mergeRegionsAsync(new byte[][] { regions.get(0).getEncodedNameAsBytes(), + regions.get(1).getEncodedNameAsBytes() }, true).get(); + await().atMost(Duration.ofSeconds(30)) + .untilAsserted(() -> assertThat(admin.getRegions(tableName0), hasSize(1))); + // take a snapshot + admin.snapshot(snapshotName0, tableName0); + // export snapshot and verify + testExportFileSystemState(tableName0, snapshotName0, snapshotName0, tableNumFiles); + } + + @TestTemplate + public void testExportFileSystemStateWithSplitRegion() throws Exception { + // disable compaction + admin.compactionSwitch(false, + admin.getRegionServers().stream().map(a -> a.getServerName()).collect(Collectors.toList())); + // create Table + String suffix = mob ? methodName + "-mob" : methodName; + TableName splitTableName = TableName.valueOf(suffix); + String splitTableSnap = "snapshot-" + suffix; + admin.createTable(TableDescriptorBuilder.newBuilder(splitTableName).setColumnFamilies( + Lists.newArrayList(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build())).build()); + + Path output = TEST_UTIL.getDataTestDir("output/cf"); + TEST_UTIL.getTestFileSystem().mkdirs(output); + // Create and load a large hfile to ensure the execution time of MR job. + HFileTestUtil.createHFile(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), + new Path(output, "test_file"), FAMILY, Bytes.toBytes("q"), Bytes.toBytes("1"), + Bytes.toBytes("9"), 9999999); + BulkLoadHFilesTool tool = new BulkLoadHFilesTool(TEST_UTIL.getConfiguration()); + tool.run(new String[] { output.getParent().toString(), splitTableName.getNameAsString() }); + + List regions = admin.getRegions(splitTableName); + assertEquals(1, regions.size()); + tableNumFiles = regions.size(); + + // split region + admin.split(splitTableName, Bytes.toBytes("5")); + await().atMost(Duration.ofSeconds(30)) + .untilAsserted(() -> assertThat(admin.getRegions(splitTableName), hasSize(2))); + + // take a snapshot + admin.snapshot(splitTableSnap, splitTableName); + // export snapshot and verify + Configuration tmpConf = TEST_UTIL.getConfiguration(); + // Decrease the buffer size of copier to avoid the export task finished shortly + tmpConf.setInt("snapshot.export.buffer.size", 1); + // Decrease the maximum files of each mapper to ensure the three files(1 hfile + 2 reference + // files) copied in different mappers concurrently. + tmpConf.setInt("snapshot.export.default.map.group", 1); + testExportFileSystemState(tmpConf, splitTableName, splitTableSnap, splitTableSnap, + tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), getHdfsDestinationDir(), false, false, + getBypassRegionPredicate(), true, false); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotMiscTestBase.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotMiscTestBase.java new file mode 100644 index 000000000000..25c469551335 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotMiscTestBase.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.SnapshotType; +import org.apache.hadoop.hbase.util.AbstractHBaseTool; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.jupiter.api.TestTemplate; + +public class ExportSnapshotMiscTestBase extends ExportSnapshotTestBase { + + protected ExportSnapshotMiscTestBase(boolean mob) { + super(mob); + } + + @TestTemplate + public void testExportWithTargetName() throws Exception { + final String targetName = "testExportWithTargetName"; + testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); + } + + @TestTemplate + public void testExportWithResetTtl() throws Exception { + String suffix = mob ? methodName + "-mob" : methodName; + TableName tableName = TableName.valueOf(suffix); + String snapshotName = "snaptb-" + suffix; + Long ttl = 100000L; + // create Table + createTable(tableName); + SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); + int tableNumFiles = admin.getRegions(tableName).size(); + // take a snapshot with TTL + Map props = new HashMap<>(); + props.put("TTL", ttl); + admin.snapshot(snapshotName, tableName, props); + Optional ttlOpt = + admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotName)) + .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny(); + assertTrue(ttlOpt.isPresent()); + assertEquals(ttl, ttlOpt.get()); + + testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, + getHdfsDestinationDir(), false, true); + } + + @TestTemplate + public void testExportExpiredSnapshot() throws Exception { + String suffix = mob ? methodName + "-mob" : methodName; + TableName tableName = TableName.valueOf(suffix); + String snapshotName = "snapshot-" + suffix; + createTable(tableName); + SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); + Map properties = new HashMap<>(); + properties.put("TTL", 10); + org.apache.hadoop.hbase.client.SnapshotDescription snapshotDescription = + new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName, + SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties); + admin.snapshot(snapshotDescription); + boolean isExist = + admin.listSnapshots().stream().anyMatch(ele -> snapshotName.equals(ele.getName())); + assertTrue(isExist); + TEST_UTIL.waitFor(60000, + () -> SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(), + snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime())); + boolean isExpiredSnapshot = + SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(), + snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()); + assertTrue(isExpiredSnapshot); + int res = runExportSnapshot(TEST_UTIL.getConfiguration(), snapshotName, snapshotName, + TEST_UTIL.getDefaultRootDirPath(), getHdfsDestinationDir(), false, false, false, true, true); + assertEquals(res, AbstractHBaseTool.EXIT_FAILURE); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotRetryTestBase.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotRetryTestBase.java new file mode 100644 index 000000000000..3ce4a3827cd8 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotRetryTestBase.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.TestTemplate; + +public class ExportSnapshotRetryTestBase extends ExportSnapshotTestBase { + + protected ExportSnapshotRetryTestBase(boolean mob) { + super(mob); + } + + /** + * Check that ExportSnapshot will succeed if something fails but the retry succeed. + */ + @TestTemplate + public void testExportRetry() throws Exception { + Path copyDir = TestExportSnapshotMisc.getLocalDestinationDir(TEST_UTIL); + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); + conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2); + conf.setInt("mapreduce.map.maxattempts", 3); + testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, + TEST_UTIL.getDefaultRootDirPath(), copyDir, true, false, getBypassRegionPredicate(), true, + false); + } + + /** + * Check that ExportSnapshot will fail if we inject failure more times than MR will retry. + */ + @TestTemplate + public void testExportFailure() throws Exception { + Path copyDir = TestExportSnapshotMisc.getLocalDestinationDir(TEST_UTIL); + FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); + copyDir = copyDir.makeQualified(fs.getUri(), fs.getWorkingDirectory()); + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); + conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4); + conf.setInt("mapreduce.map.maxattempts", 3); + testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, + TEST_UTIL.getDefaultRootDirPath(), copyDir, true, false, getBypassRegionPredicate(), false, + false); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotTestBase.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotTestBase.java new file mode 100644 index 000000000000..68e7f7eb5103 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotTestBase.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.apache.hadoop.util.ToolRunner.run; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.provider.Arguments; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; + +public class ExportSnapshotTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(ExportSnapshotTestBase.class); + + protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + protected final static byte[] FAMILY = Bytes.toBytes("cf"); + + protected String methodName; + + protected TableName tableName; + + protected String emptySnapshotName; + + protected String snapshotName; + + protected int tableNumFiles; + + protected Admin admin; + + protected boolean mob; + + public static Stream parameters() { + return Stream.of(Arguments.of(false), Arguments.of(true)); + } + + protected ExportSnapshotTestBase(boolean mob) { + this.mob = mob; + } + + @AfterAll + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniMapReduceCluster(); + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Create a table and take a snapshot of the table used by the export test. + */ + @BeforeEach + public void setUp(TestInfo testInfo) throws Exception { + this.admin = TEST_UTIL.getAdmin(); + + methodName = testInfo.getTestMethod().map(Method::getName).orElse(getClass().getSimpleName()); + + String suffix = mob ? methodName + "-mob" : methodName; + tableName = TableName.valueOf("testtb-" + suffix); + snapshotName = "snaptb0-" + suffix; + emptySnapshotName = "emptySnaptb0-" + suffix; + + // create Table + createTable(this.tableName); + + // Take an empty snapshot + admin.snapshot(emptySnapshotName, tableName); + + // Add some rows + SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); + tableNumFiles = admin.getRegions(tableName).size(); + + // take a snapshot + admin.snapshot(snapshotName, tableName); + } + + protected final void createTable(TableName tableName) throws Exception { + if (mob) { + MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY); + } else { + SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); + } + } + + protected interface RegionPredicate { + boolean evaluate(final RegionInfo regionInfo); + } + + protected final RegionPredicate getBypassRegionPredicate() { + if (mob) { + return MobUtils::isMobRegionInfo; + } else { + return null; + } + } + + @AfterEach + public void tearDown() throws Exception { + for (TableName tn : TEST_UTIL.getAdmin().listTableNames()) { + TEST_UTIL.deleteTable(tn); + } + SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin()); + SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); + } + + protected final void testExportFileSystemState(final TableName tableName, + final String snapshotName, final String targetName, int filesExpected) throws Exception { + testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, + getHdfsDestinationDir(), false); + } + + protected final void testExportFileSystemState(final TableName tableName, + final String snapshotName, final String targetName, int filesExpected, Path copyDir, + boolean overwrite) throws Exception { + testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, + overwrite, false); + } + + protected final void testExportFileSystemState(final TableName tableName, + final String snapshotName, final String targetName, int filesExpected, Path copyDir, + boolean overwrite, boolean resetTtl) throws Exception { + testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, + filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl, + getBypassRegionPredicate(), true, false); + } + + /** + * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications. + */ + protected static void testExportFileSystemState(final Configuration conf, + final TableName tableName, final String snapshotName, final String targetName, + final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite, + final boolean resetTtl, final RegionPredicate bypassregionPredicate, final boolean success, + final boolean checksumVerify) throws Exception { + FileSystem tgtFs = rawTgtDir.getFileSystem(conf); + FileSystem srcFs = srcDir.getFileSystem(conf); + Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory()); + + // Export Snapshot + int res = runExportSnapshot(conf, snapshotName, targetName, srcDir, rawTgtDir, overwrite, + resetTtl, checksumVerify, true, true); + assertEquals(success ? 0 : 1, res, "success " + success + ", res=" + res); + if (!success) { + final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName); + assertFalse(tgtFs.exists(new Path(tgtDir, targetDir)), + tgtDir.toString() + " " + targetDir.toString()); + return; + } + LOG.info("Exported snapshot"); + + // Verify File-System state + FileStatus[] rootFiles = tgtFs.listStatus(tgtDir); + assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); + for (FileStatus fileStatus : rootFiles) { + String name = fileStatus.getPath().getName(); + assertTrue(fileStatus.isDirectory(), fileStatus.toString()); + assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) + || name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY), name.toString()); + } + LOG.info("Verified filesystem state"); + + // Compare the snapshot metadata and verify the hfiles + final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName); + final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName); + verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir)); + Set snapshotFiles = + verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, resetTtl, bypassregionPredicate); + assertEquals(filesExpected, snapshotFiles.size()); + } + + /* + * verify if the snapshot folder on file-system 1 match the one on file-system 2 + */ + protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, + final FileSystem fs2, final Path root2) throws IOException { + assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); + } + + /* + * Verify if the files exists + */ + protected static Set verifySnapshot(final Configuration conf, final FileSystem fs, + final Path rootDir, final TableName tableName, final String snapshotName, + final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException { + final Path exportedSnapshot = + new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); + final Set snapshotFiles = new HashSet<>(); + final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, + new SnapshotReferenceUtil.SnapshotVisitor() { + @Override + public void storeFile(final RegionInfo regionInfo, final String family, + final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) { + return; + } + + if (!storeFile.hasReference() && !StoreFileInfo.isReference(storeFile.getName())) { + String hfile = storeFile.getName(); + snapshotFiles.add(hfile); + verifyNonEmptyFile(new Path(exportedArchive, + new Path(CommonFSUtils.getTableDir(new Path("./"), tableName), + new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); + } else { + Pair referredToRegionAndFile = + StoreFileInfo.getReferredToRegionAndFile(storeFile.getName()); + String region = referredToRegionAndFile.getFirst(); + String hfile = referredToRegionAndFile.getSecond(); + snapshotFiles.add(hfile); + verifyNonEmptyFile(new Path(exportedArchive, + new Path(CommonFSUtils.getTableDir(new Path("./"), tableName), + new Path(region, new Path(family, hfile))))); + } + } + + private void verifyNonEmptyFile(final Path path) throws IOException { + assertTrue(fs.exists(path), path + " should exists"); + assertTrue(fs.getFileStatus(path).getLen() > 0, path + " should not be empty"); + } + }); + + // Verify Snapshot description + SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); + assertTrue(desc.getName().equals(snapshotName)); + assertTrue(desc.getTable().equals(tableName.getNameAsString())); + if (resetTtl) { + assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl()); + } + return snapshotFiles; + } + + private static Set listFiles(final FileSystem fs, final Path root, final Path dir) + throws IOException { + Set files = new HashSet<>(); + LOG.debug("List files in {} in root {} at {}", fs, root, dir); + int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length(); + FileStatus[] list = CommonFSUtils.listStatus(fs, dir); + if (list != null) { + for (FileStatus fstat : list) { + LOG.debug(Objects.toString(fstat.getPath())); + if (fstat.isDirectory()) { + files.addAll(listFiles(fs, root, fstat.getPath())); + } else { + files.add(fstat.getPath().makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + .substring(rootPrefix)); + } + } + } + return files; + } + + protected final Path getHdfsDestinationDir() { + Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); + Path path = + new Path(new Path(rootDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime()); + LOG.info("HDFS export destination path: " + path); + return path; + } + + protected static Path getLocalDestinationDir(HBaseTestingUtility htu) { + Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime()); + try { + FileSystem fs = FileSystem.getLocal(htu.getConfiguration()); + LOG.info("Local export destination path: " + path); + return path.makeQualified(fs.getUri(), fs.getWorkingDirectory()); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + protected static void removeExportDir(final Path path) throws IOException { + FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); + fs.delete(path, true); + } + + protected static int runExportSnapshot(final Configuration conf, final String sourceSnapshotName, + final String targetSnapshotName, final Path srcDir, Path rawTgtDir, final boolean overwrite, + final boolean resetTtl, final boolean checksumVerify, final boolean noSourceVerify, + final boolean noTargetVerify) throws Exception { + FileSystem tgtFs = rawTgtDir.getFileSystem(conf); + FileSystem srcFs = srcDir.getFileSystem(conf); + Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory()); + LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir, + rawTgtDir, srcFs.getUri(), srcDir); + List opts = new ArrayList<>(); + opts.add("--snapshot"); + opts.add(sourceSnapshotName); + opts.add("--copy-to"); + opts.add(tgtDir.toString()); + if (!targetSnapshotName.equals(sourceSnapshotName)) { + opts.add("--target"); + opts.add(targetSnapshotName); + } + if (overwrite) { + opts.add("--overwrite"); + } + if (resetTtl) { + opts.add("--reset-ttl"); + } + if (!checksumVerify) { + opts.add("--no-checksum-verify"); + } + if (!noSourceVerify) { + opts.add("--no-source-verify"); + } + if (!noTargetVerify) { + opts.add("--no-target-verify"); + } + + // Export Snapshot + return run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()])); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotTestHelpers.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotTestHelpers.java new file mode 100644 index 000000000000..031e0819ac02 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotTestHelpers.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import java.util.UUID; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.mob.MobConstants; +import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.PermissionStorage; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; + +final class ExportSnapshotTestHelpers { + + private ExportSnapshotTestHelpers() { + } + + private static void setUpBaseConf(Configuration conf) { + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + conf.setInt("hbase.regionserver.msginterval", 100); + // If a single node has enough failures (default 3), resource manager will blacklist it. + // With only 2 nodes and tests injecting faults, we don't want that. + conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100); + conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); + } + + static void startCluster(HBaseTestingUtility util, boolean useTmpDir) throws Exception { + Configuration conf = util.getConfiguration(); + setUpBaseConf(conf); + if (useTmpDir) { + FileSystem localFs = FileSystem.getLocal(conf); + Path tmpDir = util.getDataTestDir(UUID.randomUUID().toString()) + .makeQualified(localFs.getUri(), localFs.getWorkingDirectory()); + conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, tmpDir.toUri().toString()); + } + util.startMiniCluster(1); + util.startMiniMapReduceCluster(); + } + + static void startSecureCluster(HBaseTestingUtility util) throws Exception { + setUpBaseConf(util.getConfiguration()); + // Setup separate test-data directory for MR cluster and set corresponding configurations. + // Otherwise, different test classes running MR cluster can step on each other. + util.getDataTestDir(); + + // set the always on security provider + UserProvider.setUserProviderForTesting(util.getConfiguration(), + HadoopSecurityEnabledUserProviderForTesting.class); + + // setup configuration + SecureTestUtil.enableSecurity(util.getConfiguration()); + + util.startMiniCluster(3); + util.startMiniMapReduceCluster(); + + // Wait for the ACL table to become available + util.waitTableEnabled(PermissionStorage.ACL_TABLE_NAME); + } + +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportWithChecksumTestBase.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportWithChecksumTestBase.java new file mode 100644 index 000000000000..8a8bfaca24da --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/ExportWithChecksumTestBase.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.TestTemplate; + +public class ExportWithChecksumTestBase extends ExportSnapshotTestBase { + + protected ExportWithChecksumTestBase(boolean mob) { + super(mob); + } + + @TestTemplate + public void testExportWithChecksum() throws Exception { + // Test different schemes: input scheme is hdfs:// and output scheme is file:// + // The checksum verification will fail + Path copyLocalDir = getLocalDestinationDir(TEST_UTIL); + testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, snapshotName, + tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyLocalDir, false, false, + getBypassRegionPredicate(), false, true); + + // Test same schemes: input scheme is hdfs:// and output scheme is hdfs:// + // The checksum verification will success + Path copyHdfsDir = getHdfsDestinationDir(); + testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, snapshotName, + tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyHdfsDir, false, false, + getBypassRegionPredicate(), true, true); + } + +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestConsecutiveExports.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestConsecutiveExports.java new file mode 100644 index 000000000000..cb7988922328 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestConsecutiveExports.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestConsecutiveExports extends ConsecutiveExportsTestBase { + + public TestConsecutiveExports(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, false); + } +} diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestConsecutiveExportsWithTmpDir.java similarity index 57% rename from hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java rename to hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestConsecutiveExportsWithTmpDir.java index e8127afcc709..c65e0cda8f97 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestConsecutiveExportsWithTmpDir.java @@ -15,32 +15,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hbase.client; +package org.apache.hadoop.hbase.snapshot; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; -@Category({ ClientTests.class, LargeTests.class }) -public class TestChangeSftShell extends AbstractTestShell { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestChangeSftShell.class); +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestConsecutiveExportsWithTmpDir extends ConsecutiveExportsTestBase { - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpConfig(); - - TEST_UTIL.startMiniCluster(3); - - setUpJRubyRuntime(); + public TestConsecutiveExportsWithTmpDir(boolean mob) { + super(mob); } - @Override - protected String getIncludeList() { - return "sftchange_shell_test.rb"; + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, true); } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemState.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemState.java new file mode 100644 index 000000000000..505ca7de2670 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemState.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportFileSystemState extends ExportFileSystemStateTestBase { + + public TestExportFileSystemState(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, false); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithMergeOrSplitRegion.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithMergeOrSplitRegion.java new file mode 100644 index 000000000000..ff673bb46e2d --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithMergeOrSplitRegion.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportFileSystemStateWithMergeOrSplitRegion + extends ExportFileSystemStateWithMergeOrSplitRegionTestBase { + + public TestExportFileSystemStateWithMergeOrSplitRegion(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, false); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithMergeOrSplitRegionTmpDir.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithMergeOrSplitRegionTmpDir.java new file mode 100644 index 000000000000..36c749eb126e --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithMergeOrSplitRegionTmpDir.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportFileSystemStateWithMergeOrSplitRegionTmpDir + extends ExportFileSystemStateWithMergeOrSplitRegionTestBase { + + public TestExportFileSystemStateWithMergeOrSplitRegionTmpDir(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, true); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithTmpDir.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithTmpDir.java new file mode 100644 index 000000000000..922daf9002ca --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportFileSystemStateWithTmpDir.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportFileSystemStateWithTmpDir extends ExportFileSystemStateTestBase { + + public TestExportFileSystemStateWithTmpDir(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, true); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java deleted file mode 100644 index af1d460a58f0..000000000000 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ /dev/null @@ -1,573 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import static org.apache.hadoop.util.ToolRunner.run; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.SnapshotType; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; -import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool; -import org.apache.hadoop.hbase.util.AbstractHBaseTool; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.CommonFSUtils; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.HFileTestUtil; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; - -import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; -import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; - -/** - * Test Export Snapshot Tool - */ -@Ignore // HBASE-24493 -@Category({ VerySlowMapReduceTests.class, LargeTests.class }) -public class TestExportSnapshot { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestExportSnapshot.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshot.class); - - protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - protected final static byte[] FAMILY = Bytes.toBytes("cf"); - - @Rule - public final TestName testName = new TestName(); - - protected TableName tableName; - private byte[] emptySnapshotName; - private byte[] snapshotName; - private int tableNumFiles; - private Admin admin; - - public static void setUpBaseConf(Configuration conf) { - conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - conf.setInt("hbase.regionserver.msginterval", 100); - // If a single node has enough failures (default 3), resource manager will blacklist it. - // With only 2 nodes and tests injecting faults, we don't want that. - conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100); - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - TEST_UTIL.startMiniCluster(1); - TEST_UTIL.startMiniMapReduceCluster(); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); - TEST_UTIL.shutdownMiniCluster(); - } - - /** - * Create a table and take a snapshot of the table used by the export test. - */ - @Before - public void setUp() throws Exception { - this.admin = TEST_UTIL.getAdmin(); - - tableName = TableName.valueOf("testtb-" + testName.getMethodName()); - snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName()); - emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName()); - - // create Table - createTable(this.tableName); - - // Take an empty snapshot - admin.snapshot(emptySnapshotName, tableName); - - // Add some rows - SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); - tableNumFiles = admin.getTableRegions(tableName).size(); - - // take a snapshot - admin.snapshot(snapshotName, tableName); - } - - protected void createTable(TableName tableName) throws Exception { - SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); - } - - protected interface RegionPredicate { - boolean evaluate(final RegionInfo regionInfo); - } - - protected RegionPredicate getBypassRegionPredicate() { - return null; - } - - @After - public void tearDown() throws Exception { - TEST_UTIL.deleteTable(tableName); - SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); - } - - /** - * Verify if exported snapshot and copied files matches the original one. - */ - @Test - public void testExportFileSystemState() throws Exception { - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); - } - - @Test - public void testExportFileSystemStateWithMergeRegion() throws Exception { - // disable compaction - admin.compactionSwitch(false, - admin.getRegionServers().stream().map(a -> a.getServerName()).collect(Collectors.toList())); - // create Table - TableName tableName0 = TableName.valueOf("testtb-" + testName.getMethodName() + "-1"); - byte[] snapshotName0 = Bytes.toBytes("snaptb0-" + testName.getMethodName() + "-1"); - admin.createTable( - TableDescriptorBuilder.newBuilder(tableName0) - .setColumnFamilies( - Lists.newArrayList(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build())) - .build(), - new byte[][] { Bytes.toBytes("2") }); - // put some data - try (Table table = admin.getConnection().getTable(tableName0)) { - table.put(new Put(Bytes.toBytes("1")).addColumn(FAMILY, null, Bytes.toBytes("1"))); - table.put(new Put(Bytes.toBytes("2")).addColumn(FAMILY, null, Bytes.toBytes("2"))); - } - List regions = admin.getRegions(tableName0); - assertEquals(2, regions.size()); - tableNumFiles = regions.size(); - // merge region - admin.mergeRegionsAsync(new byte[][] { regions.get(0).getEncodedNameAsBytes(), - regions.get(1).getEncodedNameAsBytes() }, true).get(); - // take a snapshot - admin.snapshot(snapshotName0, tableName0); - // export snapshot and verify - testExportFileSystemState(tableName0, snapshotName0, snapshotName0, tableNumFiles); - // delete table - TEST_UTIL.deleteTable(tableName0); - } - - @Test - public void testExportFileSystemStateWithSplitRegion() throws Exception { - // disable compaction - admin.compactionSwitch(false, admin.getRegionServers().stream().map(ServerName::getServerName) - .collect(Collectors.toList())); - // create Table - TableName splitTableName = TableName.valueOf(testName.getMethodName()); - String splitTableSnap = "snapshot-" + testName.getMethodName(); - admin.createTable(TableDescriptorBuilder.newBuilder(splitTableName).setColumnFamilies( - Lists.newArrayList(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build())).build()); - - Path output = TEST_UTIL.getDataTestDir("output/cf"); - TEST_UTIL.getTestFileSystem().mkdirs(output); - // Create and load a large hfile to ensure the execution time of MR job. - HFileTestUtil.createHFile(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), - new Path(output, "test_file"), FAMILY, Bytes.toBytes("q"), Bytes.toBytes("1"), - Bytes.toBytes("9"), 9999999); - BulkLoadHFilesTool tool = new BulkLoadHFilesTool(TEST_UTIL.getConfiguration()); - tool.run(new String[] { output.getParent().toString(), splitTableName.getNameAsString() }); - - List regions = admin.getRegions(splitTableName); - assertEquals(1, regions.size()); - tableNumFiles = regions.size(); - - // split region - admin.splitRegionAsync(regions.get(0).getEncodedNameAsBytes(), Bytes.toBytes("5")).get(); - regions = admin.getRegions(splitTableName); - assertEquals(2, regions.size()); - - // take a snapshot - admin.snapshot(splitTableSnap, splitTableName); - // export snapshot and verify - Configuration tmpConf = TEST_UTIL.getConfiguration(); - // Decrease the buffer size of copier to avoid the export task finished shortly - tmpConf.setInt("snapshot.export.buffer.size", 1); - // Decrease the maximum files of each mapper to ensure the three files(1 hfile + 2 reference - // files) copied in different mappers concurrently. - tmpConf.setInt("snapshot.export.default.map.group", 1); - testExportFileSystemState(tmpConf, splitTableName, Bytes.toBytes(splitTableSnap), - Bytes.toBytes(splitTableSnap), tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), - getHdfsDestinationDir(), false, false, getBypassRegionPredicate(), true, false); - // delete table - TEST_UTIL.deleteTable(splitTableName); - } - - @Test - public void testExportFileSystemStateWithSkipTmp() throws Exception { - TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); - try { - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); - } finally { - TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); - } - } - - @Test - public void testEmptyExportFileSystemState() throws Exception { - testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); - } - - @Test - public void testConsecutiveExports() throws Exception { - Path copyDir = getLocalDestinationDir(TEST_UTIL); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); - removeExportDir(copyDir); - } - - @Test - public void testExportWithChecksum() throws Exception { - // Test different schemes: input scheme is hdfs:// and output scheme is file:// - // The checksum verification will fail - Path copyLocalDir = getLocalDestinationDir(TEST_UTIL); - testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, snapshotName, - tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyLocalDir, false, false, - getBypassRegionPredicate(), false, true); - - // Test same schemes: input scheme is hdfs:// and output scheme is hdfs:// - // The checksum verification will success - Path copyHdfsDir = getHdfsDestinationDir(); - testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, snapshotName, - tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyHdfsDir, false, false, - getBypassRegionPredicate(), true, true); - } - - @Test - public void testExportWithTargetName() throws Exception { - final byte[] targetName = Bytes.toBytes("testExportWithTargetName"); - testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); - } - - @Test - public void testExportWithResetTtl() throws Exception { - String name = "testExportWithResetTtl"; - TableName tableName = TableName.valueOf(name); - String snapshotNameStr = "snaptb-" + name; - byte[] snapshotName = Bytes.toBytes(snapshotNameStr); - Long ttl = 100000L; - - try { - // create Table - createTable(tableName); - SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); - int tableNumFiles = admin.getRegions(tableName).size(); - // take a snapshot with TTL - Map props = new HashMap<>(); - props.put("TTL", ttl); - admin.snapshot(snapshotNameStr, tableName, props); - Optional ttlOpt = - admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotNameStr)) - .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny(); - assertTrue(ttlOpt.isPresent()); - assertEquals(ttl, ttlOpt.get()); - - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, - getHdfsDestinationDir(), false, true); - } catch (Exception e) { - throw e; - } finally { - TEST_UTIL.deleteTable(tableName); - } - } - - @Test - public void testExportExpiredSnapshot() throws Exception { - String name = "testExportExpiredSnapshot"; - TableName tableName = TableName.valueOf(name); - String snapshotName = "snapshot-" + name; - createTable(tableName); - SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); - Map properties = new HashMap<>(); - properties.put("TTL", 10); - org.apache.hadoop.hbase.client.SnapshotDescription snapshotDescription = - new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName, - SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties); - admin.snapshot(snapshotDescription); - boolean isExist = - admin.listSnapshots().stream().anyMatch(ele -> snapshotName.equals(ele.getName())); - assertTrue(isExist); - int retry = 6; - while ( - !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(), - snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0 - ) { - retry--; - Thread.sleep(10 * 1000); - } - boolean isExpiredSnapshot = - SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(), - snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()); - assertTrue(isExpiredSnapshot); - int res = runExportSnapshot(TEST_UTIL.getConfiguration(), snapshotName, snapshotName, - TEST_UTIL.getDefaultRootDirPath(), getHdfsDestinationDir(), false, false, false, true, true); - assertTrue(res == AbstractHBaseTool.EXIT_FAILURE); - } - - private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - final byte[] targetName, int filesExpected) throws Exception { - testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, - getHdfsDestinationDir(), false); - } - - protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception { - testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, - overwrite, false); - } - - protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl) - throws Exception { - testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, - filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl, - getBypassRegionPredicate(), true, false); - } - - /** - * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications. - */ - protected static void testExportFileSystemState(final Configuration conf, - final TableName tableName, final byte[] snapshotName, final byte[] targetName, - final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite, - final boolean resetTtl, final RegionPredicate bypassregionPredicate, final boolean success, - final boolean checksumVerify) throws Exception { - FileSystem tgtFs = rawTgtDir.getFileSystem(conf); - FileSystem srcFs = srcDir.getFileSystem(conf); - Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory()); - - // Export Snapshot - int res = runExportSnapshot(conf, Bytes.toString(snapshotName), Bytes.toString(targetName), - srcDir, rawTgtDir, overwrite, resetTtl, checksumVerify, true, true); - assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res); - if (!success) { - final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); - assertFalse(tgtDir.toString() + " " + targetDir.toString(), - tgtFs.exists(new Path(tgtDir, targetDir))); - return; - } - LOG.info("Exported snapshot"); - - // Verify File-System state - FileStatus[] rootFiles = tgtFs.listStatus(tgtDir); - assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); - for (FileStatus fileStatus : rootFiles) { - String name = fileStatus.getPath().getName(); - assertTrue(fileStatus.toString(), fileStatus.isDirectory()); - assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME) - || name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY)); - } - LOG.info("Verified filesystem state"); - - // Compare the snapshot metadata and verify the hfiles - final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); - final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); - verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir)); - Set snapshotFiles = verifySnapshot(conf, tgtFs, tgtDir, tableName, - Bytes.toString(targetName), resetTtl, bypassregionPredicate); - assertEquals(filesExpected, snapshotFiles.size()); - } - - /* - * verify if the snapshot folder on file-system 1 match the one on file-system 2 - */ - protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, - final FileSystem fs2, final Path root2) throws IOException { - assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); - } - - /* - * Verify if the files exists - */ - protected static Set verifySnapshot(final Configuration conf, final FileSystem fs, - final Path rootDir, final TableName tableName, final String snapshotName, - final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException { - final Path exportedSnapshot = - new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); - final Set snapshotFiles = new HashSet<>(); - final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); - SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, - new SnapshotReferenceUtil.SnapshotVisitor() { - @Override - public void storeFile(final RegionInfo regionInfo, final String family, - final SnapshotRegionManifest.StoreFile storeFile) throws IOException { - if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) { - return; - } - - if (!storeFile.hasReference() && !StoreFileInfo.isReference(storeFile.getName())) { - String hfile = storeFile.getName(); - snapshotFiles.add(hfile); - verifyNonEmptyFile(new Path(exportedArchive, - new Path(CommonFSUtils.getTableDir(new Path("./"), tableName), - new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); - } else { - Pair referredToRegionAndFile = - StoreFileInfo.getReferredToRegionAndFile(storeFile.getName()); - String region = referredToRegionAndFile.getFirst(); - String hfile = referredToRegionAndFile.getSecond(); - snapshotFiles.add(hfile); - verifyNonEmptyFile(new Path(exportedArchive, - new Path(CommonFSUtils.getTableDir(new Path("./"), tableName), - new Path(region, new Path(family, hfile))))); - } - } - - private void verifyNonEmptyFile(final Path path) throws IOException { - assertTrue(path + " should exists", fs.exists(path)); - assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); - } - }); - - // Verify Snapshot description - SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); - assertTrue(desc.getName().equals(snapshotName)); - assertTrue(desc.getTable().equals(tableName.getNameAsString())); - if (resetTtl) { - assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl()); - } - return snapshotFiles; - } - - private static Set listFiles(final FileSystem fs, final Path root, final Path dir) - throws IOException { - Set files = new HashSet<>(); - LOG.debug("List files in {} in root {} at {}", fs, root, dir); - int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length(); - FileStatus[] list = CommonFSUtils.listStatus(fs, dir); - if (list != null) { - for (FileStatus fstat : list) { - LOG.debug(Objects.toString(fstat.getPath())); - if (fstat.isDirectory()) { - files.addAll(listFiles(fs, root, fstat.getPath())); - } else { - files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix)); - } - } - } - return files; - } - - private Path getHdfsDestinationDir() { - Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - Path path = - new Path(new Path(rootDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime()); - LOG.info("HDFS export destination path: " + path); - return path; - } - - static Path getLocalDestinationDir(HBaseTestingUtility htu) { - Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime()); - try { - FileSystem fs = FileSystem.getLocal(htu.getConfiguration()); - LOG.info("Local export destination path: " + path); - return path.makeQualified(fs.getUri(), fs.getWorkingDirectory()); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - - private static void removeExportDir(final Path path) throws IOException { - FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); - fs.delete(path, true); - } - - private static int runExportSnapshot(final Configuration conf, final String sourceSnapshotName, - final String targetSnapshotName, final Path srcDir, Path rawTgtDir, final boolean overwrite, - final boolean resetTtl, final boolean checksumVerify, final boolean noSourceVerify, - final boolean noTargetVerify) throws Exception { - FileSystem tgtFs = rawTgtDir.getFileSystem(conf); - FileSystem srcFs = srcDir.getFileSystem(conf); - Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory()); - LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir, - rawTgtDir, srcFs.getUri(), srcDir); - List opts = new ArrayList<>(); - opts.add("--snapshot"); - opts.add(sourceSnapshotName); - opts.add("--copy-to"); - opts.add(tgtDir.toString()); - if (!targetSnapshotName.equals(sourceSnapshotName)) { - opts.add("--target"); - opts.add(targetSnapshotName); - } - if (overwrite) { - opts.add("--overwrite"); - } - if (resetTtl) { - opts.add("--reset-ttl"); - } - if (!checksumVerify) { - opts.add("--no-checksum-verify"); - } - if (!noSourceVerify) { - opts.add("--no-source-verify"); - } - if (!noTargetVerify) { - opts.add("--no-target-verify"); - } - - // Export Snapshot - return run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()])); - } -} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java deleted file mode 100644 index 6094f88f89fe..000000000000 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import static org.junit.Assert.assertFalse; - -import java.util.Iterator; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Tests that are adjunct to {@link TestExportSnapshot}. They used to be in same test suite but the - * test suite ran too close to the maximum time limit so we split these out. Uses facility from - * TestExportSnapshot where possible. - * @see TestExportSnapshot - */ -@Ignore // HBASE-24493 -@Category({ VerySlowMapReduceTests.class, LargeTests.class }) -public class TestExportSnapshotAdjunct { - private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshotAdjunct.class); - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestExportSnapshotAdjunct.class); - @Rule - public final TestName testName = new TestName(); - - protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - protected TableName tableName; - private String emptySnapshotName; - private String snapshotName; - private int tableNumFiles; - private Admin admin; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TestExportSnapshot.setUpBaseConf(TEST_UTIL.getConfiguration()); - TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); - } - - /** - * Check for references to '/tmp'. We are trying to avoid having references to outside of the test - * data dir when running tests. References outside of the test dir makes it so concurrent tests - * can stamp on each other by mistake. This check is for references to the 'tmp'. This is a - * strange place for this test but I want somewhere where the configuration is full -- filed w/ - * hdfs and mapreduce configurations. - */ - private void checkForReferencesToTmpDir() { - Configuration conf = TEST_UTIL.getConfiguration(); - for (Iterator> i = conf.iterator(); i.hasNext();) { - Map.Entry e = i.next(); - if (e.getKey().contains("original.hbase.dir")) { - continue; - } - if (e.getValue().contains("java.io.tmpdir")) { - continue; - } - if (e.getValue().contains("hadoop.tmp.dir")) { - continue; - } - if (e.getValue().contains("hbase.tmp.dir")) { - continue; - } - assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("tmp")); - } - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); - TEST_UTIL.shutdownMiniCluster(); - } - - /** - * Create a table and take a snapshot of the table used by the export test. - */ - @Before - public void setUp() throws Exception { - this.admin = TEST_UTIL.getAdmin(); - - tableName = TableName.valueOf("testtb-" + testName.getMethodName()); - snapshotName = "snaptb0-" + testName.getMethodName(); - emptySnapshotName = "emptySnaptb0-" + testName.getMethodName(); - - // Create Table - SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, TestExportSnapshot.FAMILY); - - // Take an empty snapshot - admin.snapshot(emptySnapshotName, tableName); - - // Add some rows - SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, TestExportSnapshot.FAMILY); - tableNumFiles = admin.getRegions(tableName).size(); - - // take a snapshot - admin.snapshot(snapshotName, tableName); - } - - @After - public void tearDown() throws Exception { - TEST_UTIL.deleteTable(tableName); - SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); - } - - /** - * Check that ExportSnapshot will succeed if something fails but the retry succeed. - */ - @Test - public void testExportRetry() throws Exception { - Path copyDir = TestExportSnapshot.getLocalDestinationDir(TEST_UTIL); - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); - conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2); - conf.setInt("mapreduce.map.maxattempts", 3); - TestExportSnapshot.testExportFileSystemState(conf, tableName, Bytes.toBytes(snapshotName), - Bytes.toBytes(snapshotName), tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, - false, null, true, false); - } - - /** - * Check that ExportSnapshot will fail if we inject failure more times than MR will retry. - */ - @Test - public void testExportFailure() throws Exception { - Path copyDir = TestExportSnapshot.getLocalDestinationDir(TEST_UTIL); - FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); - copyDir = copyDir.makeQualified(fs.getUri(), fs.getWorkingDirectory()); - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); - conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4); - conf.setInt("mapreduce.map.maxattempts", 3); - TestExportSnapshot.testExportFileSystemState(conf, tableName, Bytes.toBytes(snapshotName), - Bytes.toBytes(snapshotName), tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, - false, null, false, false); - } -} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java index 72ca0c3f7c29..79c7fb1a26c7 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.snapshot; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.Collection; @@ -26,26 +26,21 @@ import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Pair; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo; /** * Test Export Snapshot Tool helpers */ -@Category({ RegionServerTests.class, SmallTests.class }) +@Tag(MapReduceTests.TAG) +@Tag(SmallTests.TAG) public class TestExportSnapshotHelpers { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestExportSnapshotHelpers.class); - /** * Verfy the result of getBalanceSplits() method. The result are groups of files, used as input * list for the "export" mappers. All the groups should have similar amount of data. The input @@ -112,7 +107,7 @@ public void testGroupFilesForSplitsWithoutCustomFileGrouper() { Collection>> groups = inputFormat.groupFilesForSplits(conf, files); - assertEquals("Should create 3 groups", 3, groups.size()); + assertEquals(3, groups.size(), "Should create 3 groups"); long totalSize = 0; int totalFiles = 0; @@ -123,8 +118,8 @@ public void testGroupFilesForSplitsWithoutCustomFileGrouper() { } } - assertEquals("All files should be included", 10, totalFiles); - assertEquals("Total size should be preserved", 450, totalSize); + assertEquals(10, totalFiles, "All files should be included"); + assertEquals(450, totalSize, "Total size should be preserved"); } @Test @@ -146,7 +141,7 @@ public void testGroupFilesForSplitsWithCustomFileGrouper() { Collection>> groups = inputFormat.groupFilesForSplits(conf, files); - assertEquals("Should create splits based on custom grouper output", 4, groups.size()); + assertEquals(4, groups.size(), "Should create splits based on custom grouper output"); long totalSize = 0; int totalFiles = 0; @@ -157,8 +152,8 @@ public void testGroupFilesForSplitsWithCustomFileGrouper() { } } - assertEquals("All files should be included", 8, totalFiles); - assertEquals("Total size should be preserved", 140, totalSize); + assertEquals(8, totalFiles, "All files should be included"); + assertEquals(140, totalSize, "Total size should be preserved"); } @Test @@ -174,7 +169,7 @@ public void testFileLocationResolverWithNoopResolver() { new ExportSnapshot.NoopFileLocationResolver(); Set locations = resolver.getLocationsForInputFiles(files); - assertTrue("NoopFileLocationResolver should return empty locations", locations.isEmpty()); + assertTrue(locations.isEmpty(), "NoopFileLocationResolver should return empty locations"); } @Test @@ -189,13 +184,13 @@ public void testFileLocationResolverWithCustomResolver() { TestFileLocationResolver resolver = new TestFileLocationResolver(); Set locations = resolver.getLocationsForInputFiles(files); - assertEquals("Should return expected locations", 2, locations.size()); - assertTrue("Should contain rack1", locations.contains("rack1")); - assertTrue("Should contain rack2", locations.contains("rack2")); + assertEquals(2, locations.size(), "Should return expected locations"); + assertTrue(locations.contains("rack1"), "Should contain rack1"); + assertTrue(locations.contains("rack2"), "Should contain rack2"); } @Test - public void testInputSplitWithFileLocationResolver() { + public void testInputSplitWithFileLocationResolver() throws Exception { List> files = new ArrayList<>(); for (long i = 0; i < 3; i++) { SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder().setType(SnapshotFileInfo.Type.HFILE) @@ -207,26 +202,22 @@ public void testInputSplitWithFileLocationResolver() { ExportSnapshot.ExportSnapshotInputFormat.ExportSnapshotInputSplit split = new ExportSnapshot.ExportSnapshotInputFormat.ExportSnapshotInputSplit(files, resolver); - try { - String[] locations = split.getLocations(); - assertEquals("Should return 2 locations", 2, locations.length); + String[] locations = split.getLocations(); + assertEquals(2, locations.length, "Should return 2 locations"); - boolean hasRack1 = false; - boolean hasRack2 = false; - for (String location : locations) { - if ("rack1".equals(location)) { - hasRack1 = true; - } - if ("rack2".equals(location)) { - hasRack2 = true; - } + boolean hasRack1 = false; + boolean hasRack2 = false; + for (String location : locations) { + if ("rack1".equals(location)) { + hasRack1 = true; + } + if ("rack2".equals(location)) { + hasRack2 = true; } - - assertTrue("Should contain rack1", hasRack1); - assertTrue("Should contain rack2", hasRack2); - } catch (Exception e) { - throw new RuntimeException("Failed to get locations", e); } + + assertTrue(hasRack1, "Should contain rack1"); + assertTrue(hasRack2, "Should contain rack2"); } public static class TestCustomFileGrouper implements ExportSnapshot.CustomFileGrouper { diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotMisc.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotMisc.java new file mode 100644 index 000000000000..8ac6f4aa0201 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotMisc.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportSnapshotMisc extends ExportSnapshotMiscTestBase { + + protected TestExportSnapshotMisc(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, true); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotMiscWithTmpDir.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotMiscWithTmpDir.java new file mode 100644 index 000000000000..743bc37cae9e --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotMiscWithTmpDir.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportSnapshotMiscWithTmpDir extends ExportSnapshotMiscTestBase { + + protected TestExportSnapshotMiscWithTmpDir(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, false); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotRetry.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotRetry.java new file mode 100644 index 000000000000..09f4a3b01ef7 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotRetry.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportSnapshotRetry extends ExportSnapshotRetryTestBase { + + public TestExportSnapshotRetry(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, false); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotRetryWithTmpDir.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotRetryWithTmpDir.java new file mode 100644 index 000000000000..144b2228d753 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotRetryWithTmpDir.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportSnapshotRetryWithTmpDir extends ExportSnapshotRetryTestBase { + + public TestExportSnapshotRetryWithTmpDir(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, true); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java index cd24787e016d..e38d797345a8 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.snapshot; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.HashSet; @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -33,14 +32,12 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,25 +46,24 @@ * separate the tests. See companion file for test of v2 snapshot. * @see TestExportSnapshotV2NoCluster */ -@Category({ MapReduceTests.class, MediumTests.class }) +@Tag(MapReduceTests.TAG) +@Tag(SmallTests.TAG) public class TestExportSnapshotV1NoCluster { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestExportSnapshotV1NoCluster.class); + private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshotV1NoCluster.class); private HBaseCommonTestingUtility testUtil = new HBaseCommonTestingUtility(); private Path testDir; private FileSystem fs; - @Before + @BeforeEach public void setUpBefore() throws Exception { // Make sure testDir is on LocalFileSystem this.fs = FileSystem.getLocal(this.testUtil.getConfiguration()); this.testDir = setup(fs, this.testUtil); LOG.info("fs={}, fsuri={}, fswd={}, testDir={}", this.fs, this.fs.getUri(), this.fs.getWorkingDirectory(), this.testDir); - assertTrue("FileSystem '" + fs + "' is not local", fs instanceof LocalFileSystem); + assertTrue(fs instanceof LocalFileSystem, "FileSystem '" + fs + "' is not local"); } /** @@ -122,9 +118,9 @@ static void testSnapshotWithRefsExportFileSystemState(FileSystem fs, } } int snapshotFilesCount = dataFiles.size(); - byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); + String snapshotName = builder.getSnapshotDescription().getName(); TableName tableName = builder.getTableDescriptor().getTableName(); - TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(), tableName, + ExportSnapshotTestBase.testExportFileSystemState(testUtil.getConfiguration(), tableName, snapshotName, snapshotName, snapshotFilesCount, testDir, getDestinationDir(fs, testUtil, testDir), false, false, null, true, false); } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV2NoCluster.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV2NoCluster.java index 70c598d520f6..98ec2263949e 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV2NoCluster.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV2NoCluster.java @@ -17,20 +17,18 @@ */ package org.apache.hadoop.hbase.snapshot; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,25 +36,22 @@ * Test Export Snapshot Tool; tests v2 snapshots. * @see TestExportSnapshotV1NoCluster */ -@Category({ MapReduceTests.class, MediumTests.class }) +@Tag(MapReduceTests.TAG) +@Tag(SmallTests.TAG) public class TestExportSnapshotV2NoCluster { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestExportSnapshotV2NoCluster.class); - private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshotV2NoCluster.class); private HBaseCommonTestingUtility testUtil = new HBaseCommonTestingUtility(); private Path testDir; private FileSystem fs; - @Before + @BeforeEach public void before() throws Exception { // Make sure testDir is on LocalFileSystem this.fs = FileSystem.getLocal(this.testUtil.getConfiguration()); this.testDir = TestExportSnapshotV1NoCluster.setup(this.fs, this.testUtil); LOG.info("fs={}, testDir={}", this.fs, this.testDir); - assertTrue("FileSystem '" + fs + "' is not local", fs instanceof LocalFileSystem); + assertTrue(fs instanceof LocalFileSystem, "FileSystem '" + fs + "' is not local"); } @Test diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotWithTemporaryDirectory.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotWithTemporaryDirectory.java deleted file mode 100644 index fe380e683db0..000000000000 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotWithTemporaryDirectory.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import java.io.IOException; -import java.util.UUID; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.experimental.categories.Category; - -@Ignore // HBASE-24493 -@Category({ MediumTests.class }) -public class TestExportSnapshotWithTemporaryDirectory extends TestExportSnapshot { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestExportSnapshotWithTemporaryDirectory.class); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TestExportSnapshot.tearDownAfterClass(); - } - - public static void setUpBaseConf(Configuration conf) { - Path tmpDir = null; - try { - FileSystem localFs = FileSystem.getLocal(conf); - tmpDir = TEST_UTIL.getDataTestDir(UUID.randomUUID().toString()) - .makeQualified(localFs.getUri(), localFs.getWorkingDirectory()); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - TestExportSnapshot.setUpBaseConf(conf); - conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, tmpDir.toUri().toString()); - } -} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportWithChecksum.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportWithChecksum.java new file mode 100644 index 000000000000..710673727400 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportWithChecksum.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportWithChecksum extends ExportWithChecksumTestBase { + + public TestExportWithChecksum(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, false); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportWithChecksumWithTmpDir.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportWithChecksumWithTmpDir.java new file mode 100644 index 000000000000..bed72735ddea --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportWithChecksumWithTmpDir.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestExportWithChecksumWithTmpDir extends ExportWithChecksumTestBase { + + public TestExportWithChecksumWithTmpDir(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startCluster(TEST_UTIL, true); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java deleted file mode 100644 index 4943b40d6a71..000000000000 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.experimental.categories.Category; - -/** - * Test Export Snapshot Tool - */ -@Ignore // HBASE-24493 -@Category({ VerySlowRegionServerTests.class, LargeTests.class }) -public class TestMobExportSnapshot extends TestExportSnapshot { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMobExportSnapshot.class); - - public static void setUpBaseConf(Configuration conf) { - TestExportSnapshot.setUpBaseConf(conf); - conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); - } - - @Override - protected void createTable(TableName tableName) throws Exception { - MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY); - } - - @Override - protected RegionPredicate getBypassRegionPredicate() { - return new RegionPredicate() { - @Override - public boolean evaluate(final RegionInfo regionInfo) { - return MobUtils.isMobRegionInfo(regionInfo); - } - }; - } -} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java deleted file mode 100644 index 2fa686f768f1..000000000000 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.PermissionStorage; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; - -/** - * Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode. - */ -@Category({ VerySlowRegionServerTests.class, LargeTests.class }) -public class TestMobSecureExportSnapshot extends TestMobExportSnapshot { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMobSecureExportSnapshot.class); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - // Setup separate test-data directory for MR cluster and set corresponding configurations. - // Otherwise, different test classes running MR cluster can step on each other. - TEST_UTIL.getDataTestDir(); - - // set the always on security provider - UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), - HadoopSecurityEnabledUserProviderForTesting.class); - - // setup configuration - SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); - - TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); - - // Wait for the ACL table to become available - TEST_UTIL.waitTableEnabled(PermissionStorage.ACL_TABLE_NAME); - } -} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureConsecutiveExports.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureConsecutiveExports.java new file mode 100644 index 000000000000..993cc0370eb6 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureConsecutiveExports.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestSecureConsecutiveExports extends ConsecutiveExportsTestBase { + + public TestSecureConsecutiveExports(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startSecureCluster(TEST_UTIL); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportFileSystemState.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportFileSystemState.java new file mode 100644 index 000000000000..ab4d2659f2e8 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportFileSystemState.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestSecureExportFileSystemState extends ExportFileSystemStateTestBase { + + public TestSecureExportFileSystemState(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startSecureCluster(TEST_UTIL); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportFileSystemStateWithMergeOrSplitRegion.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportFileSystemStateWithMergeOrSplitRegion.java new file mode 100644 index 000000000000..45fcb743b3ed --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportFileSystemStateWithMergeOrSplitRegion.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestSecureExportFileSystemStateWithMergeOrSplitRegion + extends ExportFileSystemStateWithMergeOrSplitRegionTestBase { + + public TestSecureExportFileSystemStateWithMergeOrSplitRegion(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startSecureCluster(TEST_UTIL); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java deleted file mode 100644 index a2a588ac5724..000000000000 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.PermissionStorage; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; - -/** - * Reruns TestExportSnapshot using ExportSnapshot in secure mode. - */ -@Category({ VerySlowRegionServerTests.class, LargeTests.class }) -public class TestSecureExportSnapshot extends TestExportSnapshot { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSecureExportSnapshot.class); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - // Setup separate test-data directory for MR cluster and set corresponding configurations. - // Otherwise, different test classes running MR cluster can step on each other. - TEST_UTIL.getDataTestDir(); - - // set the always on security provider - UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), - HadoopSecurityEnabledUserProviderForTesting.class); - - // setup configuration - SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); - - TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); - - // Wait for the ACL table to become available - TEST_UTIL.waitTableEnabled(PermissionStorage.ACL_TABLE_NAME); - } -} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshotMisc.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshotMisc.java new file mode 100644 index 000000000000..088499e5f7d6 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshotMisc.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestSecureExportSnapshotMisc extends ExportSnapshotMiscTestBase { + + protected TestSecureExportSnapshotMisc(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startSecureCluster(TEST_UTIL); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshotRetry.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshotRetry.java new file mode 100644 index 000000000000..ad32fee09e26 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshotRetry.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestSecureExportSnapshotRetry extends ExportSnapshotRetryTestBase { + + public TestSecureExportSnapshotRetry(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startSecureCluster(TEST_UTIL); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportWithChecksum.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportWithChecksum.java new file mode 100644 index 000000000000..0f6d5a125015 --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportWithChecksum.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(MapReduceTests.TAG) +@Tag(LargeTests.TAG) +@HBaseParameterizedTestTemplate(name = "{index}: mob = {0}") +public class TestSecureExportWithChecksum extends ExportWithChecksumTestBase { + + public TestSecureExportWithChecksum(boolean mob) { + super(mob); + } + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + ExportSnapshotTestHelpers.startSecureCluster(TEST_UTIL); + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java index 314ee93bff5d..28b0a24b4045 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java @@ -172,6 +172,10 @@ public class LoadTestTool extends AbstractHBaseTool { protected static final String OPT_REGION_REPLICA_ID_USAGE = "Region replica id to do the reads from"; + public static final String OPT_TIMELINE_CONSISTENCY = "timeline"; + protected static final String OPT_TIMELINE_CONSISTENCY_USAGE = + "Use TIMELINE consistency in read operations. Leave region_replica_id unset, otherwise it will override this setting."; + public static final String OPT_MOB_THRESHOLD = "mob_threshold"; protected static final String OPT_MOB_THRESHOLD_USAGE = "Desired cell size to exceed in bytes that will use the MOB write path"; @@ -227,6 +231,7 @@ public class LoadTestTool extends AbstractHBaseTool { private int numRegionsPerServer = DEFAULT_NUM_REGIONS_PER_SERVER; private int regionReplication = -1; // not set private int regionReplicaId = -1; // not set + private boolean timelineConsistency = false; private int mobThreshold = -1; // not set @@ -360,6 +365,7 @@ protected void addOptions() { addOptWithArg(OPT_NUM_REGIONS_PER_SERVER, OPT_NUM_REGIONS_PER_SERVER_USAGE); addOptWithArg(OPT_REGION_REPLICATION, OPT_REGION_REPLICATION_USAGE); addOptWithArg(OPT_REGION_REPLICA_ID, OPT_REGION_REPLICA_ID_USAGE); + addOptNoArg(OPT_TIMELINE_CONSISTENCY, OPT_TIMELINE_CONSISTENCY_USAGE); addOptWithArg(OPT_MOB_THRESHOLD, OPT_MOB_THRESHOLD_USAGE); } @@ -519,6 +525,11 @@ protected void processOptions(CommandLine cmd) { if (cmd.hasOption(OPT_REGION_REPLICA_ID)) { regionReplicaId = Integer.parseInt(cmd.getOptionValue(OPT_REGION_REPLICA_ID)); } + + timelineConsistency = false; + if (cmd.hasOption(OPT_TIMELINE_CONSISTENCY)) { + timelineConsistency = true; + } } private void parseColumnFamilyOptions(CommandLine cmd) { @@ -705,6 +716,7 @@ protected int loadTable() throws IOException { readerThreads.setKeyWindow(keyWindow); readerThreads.setMultiGetBatchSize(multiGetBatchSize); readerThreads.setRegionReplicaId(regionReplicaId); + readerThreads.setTimelineConsistency(timelineConsistency); } if (isUpdate && isWrite) { diff --git a/hbase-metrics-api/pom.xml b/hbase-metrics-api/pom.xml index 9c8d9dc79b4a..c1a5af10d414 100644 --- a/hbase-metrics-api/pom.xml +++ b/hbase-metrics-api/pom.xml @@ -74,8 +74,18 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java b/hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java index b9df823e7565..83cdcbb2fca6 100644 --- a/hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java +++ b/hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java @@ -17,30 +17,24 @@ */ package org.apache.hadoop.hbase.metrics; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.mock; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; /** * Test class for {@link MetricRegistriesLoader}. */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestMetricRegistriesLoader { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricRegistriesLoader.class); - @Test - public void testLoadSinleInstance() { + public void testLoadSingleInstance() { MetricRegistries loader = mock(MetricRegistries.class); MetricRegistries instance = MetricRegistriesLoader.load(Lists.newArrayList(loader)); assertEquals(loader, instance); diff --git a/hbase-metrics/pom.xml b/hbase-metrics/pom.xml index 5fe3d26147ff..bfb422ae44af 100644 --- a/hbase-metrics/pom.xml +++ b/hbase-metrics/pom.xml @@ -83,8 +83,18 @@ true - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java index 0d86e2aeb496..317c3f74550c 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java @@ -17,29 +17,23 @@ */ package org.apache.hadoop.hbase.metrics.impl; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.metrics.Counter; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test class for {@link CounterImpl}. */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestCounterImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCounterImpl.class); - private Counter counter; - @Before + @BeforeEach public void setup() { this.counter = new CounterImpl(); } diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java index 9af666b095f1..2473dab5b3e7 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java @@ -18,27 +18,21 @@ package org.apache.hadoop.hbase.metrics.impl; import com.codahale.metrics.Meter; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; /** * Test class for {@link DropwizardMeter}. */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestDropwizardMeter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestDropwizardMeter.class); - private Meter meter; - @Before + @BeforeEach public void setup() { this.meter = Mockito.mock(Meter.class); } diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java index ca5c6a476685..1ad600912880 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java @@ -17,29 +17,24 @@ */ package org.apache.hadoop.hbase.metrics.impl; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; import java.util.Random; import java.util.concurrent.ThreadLocalRandom; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Testcases for FastLongHistogram. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestFastLongHistogram { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFastLongHistogram.class); - private static void doTestUniform(FastLongHistogram hist) { long[] VALUES = { 0, 10, 20, 30, 40, 50 }; double[] qs = new double[VALUES.length]; @@ -54,8 +49,8 @@ private static void doTestUniform(FastLongHistogram hist) { long[] vals = hist.getQuantiles(qs); System.out.println(Arrays.toString(vals)); for (int j = 0; j < qs.length; j++) { - Assert.assertTrue(j + "-th element org: " + VALUES[j] + ", act: " + vals[j], - Math.abs(vals[j] - VALUES[j]) <= 10); + assertTrue(Math.abs(vals[j] - VALUES[j]) <= 10, + j + "-th element org: " + VALUES[j] + ", act: " + vals[j]); } hist.snapshotAndReset(); } @@ -87,9 +82,9 @@ public void testAdaptionOfChange() { long[] vals = hist.getQuantiles(new double[] { 0.25, 0.75, 0.95 }); System.out.println(Arrays.toString(vals)); if (n == 0) { - Assert.assertTrue("Out of possible value", vals[0] >= 0 && vals[0] <= 50); - Assert.assertTrue("Out of possible value", vals[1] >= 50 && vals[1] <= 100); - Assert.assertTrue("Out of possible value", vals[2] >= 900 && vals[2] <= 1100); + assertTrue(vals[0] >= 0 && vals[0] <= 50, "Out of possible value"); + assertTrue(vals[1] >= 50 && vals[1] <= 100, "Out of possible value"); + assertTrue(vals[2] >= 900 && vals[2] <= 1100, "Out of possible value"); } hist.snapshotAndReset(); diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java index e1ed9cf6a5be..ccdc925c1fd3 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java @@ -17,33 +17,26 @@ */ package org.apache.hadoop.hbase.metrics.impl; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.metrics.Gauge; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test class for {@link Gauge}. */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestGauge { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestGauge.class); - @Test public void testGetValue() { SimpleGauge gauge = new SimpleGauge(); - assertEquals(0, (long) gauge.getValue()); gauge.setValue(1000L); - assertEquals(1000L, (long) gauge.getValue()); } diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java index 70d9598570d6..87bebd684f55 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java @@ -17,30 +17,23 @@ */ package org.apache.hadoop.hbase.metrics.impl; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.stream.IntStream; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.metrics.Snapshot; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test case for {@link HistogramImpl} */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestHistogramImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHistogramImpl.class); - @Test public void testUpdate() { HistogramImpl histogram = new HistogramImpl(); - assertEquals(0, histogram.getCount()); histogram.update(0); @@ -51,7 +44,6 @@ public void testUpdate() { histogram.update(20); histogram.update(30); - assertEquals(4, histogram.getCount()); } @@ -61,7 +53,6 @@ public void testSnapshot() { IntStream.range(0, 100).forEach(histogram::update); Snapshot snapshot = histogram.snapshot(); - assertEquals(100, snapshot.getCount()); assertEquals(49, snapshot.getMedian()); assertEquals(49, snapshot.getMean()); @@ -100,9 +91,7 @@ public void testSnapshot() { assertEquals(199, snapshot.get999thPercentile()); IntStream.range(500, 1000).forEach(histogram::update); - snapshot = histogram.snapshot(); - assertEquals(500, snapshot.getCount()); assertEquals(749, snapshot.getMedian()); assertEquals(749, snapshot.getMean()); @@ -115,6 +104,5 @@ public void testSnapshot() { assertEquals(989, snapshot.get98thPercentile()); assertEquals(994, snapshot.get99thPercentile()); assertEquals(998, snapshot.get999thPercentile()); - } } diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistriesImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistriesImpl.java index e431ee86ceae..5852104d7096 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistriesImpl.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistriesImpl.java @@ -17,28 +17,23 @@ */ package org.apache.hadoop.hbase.metrics.impl; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Optional; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.metrics.MetricRegistries; import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test class for {@link MetricRegistries}. */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestMetricRegistriesImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricRegistriesImpl.class); - @Test public void testMetricsRegistriesRemoveRef() { MetricRegistryInfo registryInfo = @@ -57,6 +52,6 @@ public void testMetricsRegistriesRemoveRef() { MetricRegistries.global().remove(registryInfo); Optional registry4 = MetricRegistries.global().get(registryInfo); - assertTrue(!registry4.isPresent()); + assertFalse(registry4.isPresent()); } } diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java index 8cae06361204..f61d6eebf9a9 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hbase.metrics.impl; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Map; import java.util.Optional; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.metrics.Counter; import org.apache.hadoop.hbase.metrics.Gauge; import org.apache.hadoop.hbase.metrics.Meter; @@ -32,22 +31,17 @@ import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; import org.apache.hadoop.hbase.metrics.Timer; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestMetricRegistryImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMetricRegistryImpl.class); - private MetricRegistryInfo info; private MetricRegistryImpl registry; - @Before + @BeforeEach public void setUp() { info = new MetricRegistryInfo("foo", "bar", "baz", "foobar", false); registry = new MetricRegistryImpl(info); @@ -112,8 +106,8 @@ public void testRegister() { @Test public void testDoubleRegister() { - Gauge g1 = registry.register("mygauge", () -> 42L); - Gauge g2 = registry.register("mygauge", () -> 52L); + Gauge g1 = registry.register("mygauge", () -> 42L); + Gauge g2 = registry.register("mygauge", () -> 52L); // second gauge is ignored if it exists assertEquals(g1, g2); @@ -132,7 +126,7 @@ public void testDoubleRegister() { public void testGetMetrics() { CounterImpl counter = new CounterImpl(); registry.register("mycounter", counter); - Gauge gauge = registry.register("mygauge", () -> 42L); + Gauge gauge = registry.register("mygauge", () -> 42L); Timer timer = registry.timer("mytimer"); Map metrics = registry.getMetrics(); diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java index 2d29ff17943b..cbfbbe122abb 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java @@ -17,32 +17,26 @@ */ package org.apache.hadoop.hbase.metrics.impl; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Collection; import java.util.Set; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestRefCountingMap { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRefCountingMap.class); - private RefCountingMap map; - @Before + @BeforeEach public void setUp() { map = new RefCountingMap<>(); } diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java index 1bfa02fc4b64..3b3eded1ea9e 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java @@ -17,30 +17,24 @@ */ package org.apache.hadoop.hbase.metrics.impl; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.metrics.Timer; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test class for {@link TimerImpl} */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestTimerImpl { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTimerImpl.class); - private Timer timer; - @Before + @BeforeEach public void setup() { this.timer = new TimerImpl(); } @@ -52,7 +46,6 @@ public void testUpdate() { timer.update(42, TimeUnit.MILLISECONDS); assertEquals(3, timer.getHistogram().getCount()); assertEquals(3, timer.getMeter().getCount()); - assertEquals(TimeUnit.MILLISECONDS.toMicros(41), timer.getHistogram().snapshot().getMedian()); } } diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml index caa037c21510..046440a85bd0 100644 --- a/hbase-procedure/pom.xml +++ b/hbase-procedure/pom.xml @@ -77,8 +77,18 @@ true - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java index 32be56e44dbd..1efab6f5d3a8 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -264,8 +267,8 @@ public static void setKillAndToggleBeforeStoreUpdateInRollback( if ( procExecutor.testing.killBeforeStoreUpdate || procExecutor.testing.toggleKillBeforeStoreUpdate ) { - assertEquals("expected only one executor running during test with kill/restart", 1, - procExecutor.getCorePoolSize()); + assertEquals(1, procExecutor.getCorePoolSize(), + "expected only one executor running during test with kill/restart"); } } @@ -342,48 +345,48 @@ public static void waitNoProcedureRunning(ProcedureExecutor procExe public static void assertProcNotYetCompleted(ProcedureExecutor procExecutor, long procId) { - assertFalse("expected a running proc", procExecutor.isFinished(procId)); - assertEquals(null, procExecutor.getResult(procId)); + assertFalse(procExecutor.isFinished(procId), "expected a running proc"); + assertNull(procExecutor.getResult(procId)); } public static void assertProcNotFailed(ProcedureExecutor procExecutor, long procId) { Procedure result = procExecutor.getResult(procId); - assertTrue("expected procedure result", result != null); + assertNotNull(result, "expected procedure result"); assertProcNotFailed(result); } public static void assertProcNotFailed(final Procedure result) { - assertFalse("found exception: " + result.getException(), result.isFailed()); + assertFalse(result.isFailed(), "found exception: " + result.getException()); } public static Throwable assertProcFailed(final ProcedureExecutor procExecutor, final long procId) { Procedure result = procExecutor.getResult(procId); - assertTrue("expected procedure result", result != null); + assertNotNull(result, "expected procedure result"); return assertProcFailed(result); } public static Throwable assertProcFailed(final Procedure result) { - assertEquals(true, result.isFailed()); + assertTrue(result.isFailed()); LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage()); return getExceptionCause(result); } public static void assertIsAbortException(final Procedure result) { Throwable cause = assertProcFailed(result); - assertTrue("expected abort exception, got " + cause, - cause instanceof ProcedureAbortedException); + assertInstanceOf(ProcedureAbortedException.class, cause, + "expected abort exception, got " + cause); } public static void assertIsTimeoutException(final Procedure result) { Throwable cause = assertProcFailed(result); - assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException); + assertInstanceOf(TimeoutIOException.class, cause, "expected TimeoutIOException, got " + cause); } public static void assertIsIllegalArgumentException(final Procedure result) { Throwable cause = assertProcFailed(result); - assertTrue("expected IllegalArgumentIOException, got " + cause, - cause instanceof IllegalArgumentIOException); + assertInstanceOf(IllegalArgumentIOException.class, cause, + "expected IllegalArgumentIOException, got " + cause); } public static Throwable getExceptionCause(final Procedure procInfo) { @@ -669,7 +672,7 @@ public void load(ProcedureIterator procIter) throws IOException { runnable.add(proc); } if (procIds != null) { - assertTrue("procId=" + procId + " unexpected", procIds.contains(procId)); + assertTrue(procIds.contains(procId), "procId=" + procId + " unexpected"); } } } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestChildProcedures.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestChildProcedures.java index b8642b675f5a..011fb20491ab 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestChildProcedures.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestChildProcedures.java @@ -17,32 +17,27 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestChildProcedures { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestChildProcedures.class); - private static final Logger LOG = LoggerFactory.getLogger(TestChildProcedures.class); private static final int PROCEDURE_EXECUTOR_SLOTS = 1; @@ -56,7 +51,7 @@ public class TestChildProcedures { private Path testDir; private Path logDir; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); @@ -72,7 +67,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -88,7 +83,7 @@ public void testChildLoad() throws Exception { ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); - assertTrue("expected completed proc", procExecutor.isFinished(procId)); + assertTrue(procExecutor.isFinished(procId), "expected completed proc"); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); } @@ -105,7 +100,7 @@ public void testChildLoadWithSteppedRestart() throws Exception { restartCount++; } assertEquals(3, restartCount); - assertTrue("expected completed proc", procExecutor.isFinished(procId)); + assertTrue(procExecutor.isFinished(procId), "expected completed proc"); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); } @@ -126,7 +121,7 @@ public void testChildLoadWithRestartAfterChildSuccess() throws Exception { restartCount++; } assertEquals(4, restartCount); - assertTrue("expected completed proc", procExecutor.isFinished(procId)); + assertTrue(procExecutor.isFinished(procId), "expected completed proc"); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); } @@ -161,9 +156,9 @@ public void testChildRollbackLoadWithSteppedRestart() throws Exception { } private void assertProcFailed(long procId) { - assertTrue("expected completed proc", procExecutor.isFinished(procId)); + assertTrue(procExecutor.isFinished(procId), "expected completed proc"); Procedure result = procExecutor.getResult(procId); - assertEquals(true, result.isFailed()); + assertTrue(result.isFailed()); LOG.info(result.getException().getMessage()); } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestForceUpdateProcedure.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestForceUpdateProcedure.java index e3965a51aa63..044c228a9573 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestForceUpdateProcedure.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestForceUpdateProcedure.java @@ -17,62 +17,55 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Exchanger; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestForceUpdateProcedure { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestForceUpdateProcedure.class); - private static HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility(); private static WALProcedureStore STORE; private static ProcedureExecutor EXEC; - private static Exchanger EXCHANGER = new Exchanger<>(); + private static final Exchanger EXCHANGER = new Exchanger<>(); - private static int WAL_COUNT = 5; + private static final int WAL_COUNT = 5; - @Rule - public final TestName name = new TestName(); + private String methodName; private void createStoreAndExecutor() throws IOException { UTIL.getConfiguration().setInt(CompletedProcedureCleaner.CLEANER_INTERVAL_CONF_KEY, 1000); - Path logDir = UTIL.getDataTestDir(name.getMethodName()); + Path logDir = UTIL.getDataTestDir(methodName); STORE = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), logDir); STORE.start(1); - EXEC = new ProcedureExecutor(UTIL.getConfiguration(), null, STORE); + EXEC = new ProcedureExecutor<>(UTIL.getConfiguration(), null, STORE); ProcedureTestingUtility.initAndStartWorkers(EXEC, 1, true); } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws IOException { UTIL.getConfiguration().setInt(WALProcedureStore.WAL_COUNT_WARN_THRESHOLD_CONF_KEY, WAL_COUNT); } @@ -84,17 +77,18 @@ private void stopStoreAndExecutor() { STORE = null; } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws IOException { UTIL.cleanupTestDir(); } - @Before - public void setUp() throws IOException { + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException { + methodName = testInfo.getTestMethod().get().getName(); createStoreAndExecutor(); } - @After + @AfterEach public void tearDown() { stopStoreAndExecutor(); } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestLockAndQueue.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestLockAndQueue.java index 58c3bb3277ae..e0374f461abf 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestLockAndQueue.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestLockAndQueue.java @@ -17,26 +17,21 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashMap; import java.util.Map; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestLockAndQueue { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLockAndQueue.class); - @Test public void testHasLockAccess() { Map> procMap = new HashMap<>(); diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java index f4ec5a2ad56a..ac2152f46b3c 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureBypass.java @@ -17,35 +17,30 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureBypass { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureBypass.class); - private static final Logger LOG = LoggerFactory.getLogger(TestProcedureBypass.class); private static final int PROCEDURE_EXECUTOR_SLOTS = 1; @@ -64,7 +59,7 @@ public class TestProcedureBypass { private static class TestProcEnv { } - @BeforeClass + @BeforeAll public static void setUp() throws Exception { htu = new HBaseCommonTestingUtility(); @@ -81,6 +76,13 @@ public static void setUp() throws Exception { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } + @AfterAll + public static void tearDown() throws Exception { + procExecutor.stop(); + procStore.stop(false); + procExecutor.join(); + } + @Test public void testBypassSuspendProcedure() throws Exception { final SuspendProcedure proc = new SuspendProcedure(); @@ -157,13 +159,6 @@ public void testBypassingWaitingTimeoutProcedures() throws Exception { LOG.info("{} finished", proc); } - @AfterClass - public static void tearDown() throws Exception { - procExecutor.stop(); - procStore.stop(false); - procExecutor.join(); - } - public static class SuspendProcedure extends ProcedureTestingUtility.NoopProcedure { public SuspendProcedure() { @@ -284,5 +279,4 @@ protected int getStateId(StuckStateMachineState tState) { return tState.ordinal(); } } - } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureCleanup.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureCleanup.java index f54af62ef70b..5450e12a16bb 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureCleanup.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureCleanup.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Arrays; @@ -28,29 +28,24 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureCleanup { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureCleanup.class); - private static final Logger LOG = LoggerFactory.getLogger(TestProcedureCleanup.class); private static final int PROCEDURE_EXECUTOR_SLOTS = 2; @@ -65,18 +60,17 @@ public class TestProcedureCleanup { private static Path testDir; private static Path logDir; - @Rule - public final TestName name = new TestName(); + private String methodName; private void createProcExecutor() throws Exception { - logDir = new Path(testDir, name.getMethodName()); + logDir = new Path(testDir, methodName); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), null, procStore); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true, true); } - @BeforeClass + @BeforeAll public static void setUp() throws Exception { htu = new HBaseCommonTestingUtility(); htu.getConfiguration().setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, true); @@ -86,6 +80,11 @@ public static void setUp() throws Exception { assertTrue(testDir.depth() > 1); } + @BeforeEach + public void setUpEach(TestInfo testInfo) throws Exception { + methodName = testInfo.getTestMethod().get().getName(); + } + @Test public void testProcedureShouldNotCleanOnLoad() throws Exception { createProcExecutor(); diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java index 039b25d347ea..ffbdab31ddba 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java @@ -17,23 +17,21 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,13 +39,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; -@Category({ MasterTests.class, MediumTests.class }) +@Tag(MasterTests.TAG) +@Tag(MediumTests.TAG) public class TestProcedureEvents { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureEvents.class); - private static final Logger LOG = LoggerFactory.getLogger(TestProcedureEvents.class); private TestProcEnv procEnv; @@ -58,7 +53,7 @@ public class TestProcedureEvents { private FileSystem fs; private Path logDir; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); Path testDir = htu.getDataTestDir(); @@ -72,7 +67,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, 1, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -174,7 +169,7 @@ protected Procedure[] execute(final TestProcEnv env) throws ProcedureSuspendedEx @Override protected synchronized boolean setTimeoutFailure(final TestProcEnv env) { int n = ntimeouts.incrementAndGet(); - LOG.info("HANDLE TIMEOUT " + this + " ntimeouts=" + n); + LOG.info("HANDLE TIMEOUT {} ntimeouts={}", this, n); setState(ProcedureState.RUNNABLE); event.wake((AbstractProcedureScheduler) env.getProcedureScheduler()); return false; diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java index a89f0f64ca79..dbb56b4165ba 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -26,27 +27,23 @@ import java.util.Objects; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureExecution { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureExecution.class); private static final Logger LOG = LoggerFactory.getLogger(TestProcedureExecution.class); @@ -61,7 +58,7 @@ public class TestProcedureExecution { private Path testDir; private Path logDir; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); @@ -75,7 +72,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -149,10 +146,10 @@ public void testBadSubprocList() { // failed state with 2 execute and 2 rollback LOG.info(Objects.toString(state)); Procedure result = procExecutor.getResult(rootId); - assertTrue(state.toString(), result.isFailed()); + assertTrue(result.isFailed(), state.toString()); ProcedureTestingUtility.assertIsIllegalArgumentException(result); - assertEquals(state.toString(), 4, state.size()); + assertEquals(4, state.size(), state.toString()); assertEquals("rootProc-execute", state.get(0)); assertEquals("subProc1-execute", state.get(1)); assertEquals("subProc1-rollback", state.get(2)); @@ -171,7 +168,7 @@ public void testSingleSequentialProc() { LOG.info(Objects.toString(state)); Procedure result = procExecutor.getResult(rootId); ProcedureTestingUtility.assertProcNotFailed(result); - assertEquals(state.toString(), 3, state.size()); + assertEquals(3, state.size(), state.toString()); } @Test @@ -186,13 +183,13 @@ public void testSingleSequentialProcRollback() { // the 3rd proc fail, rollback after 2 successful execution LOG.info(Objects.toString(state)); Procedure result = procExecutor.getResult(rootId); - assertTrue(state.toString(), result.isFailed()); + assertTrue(result.isFailed(), state.toString()); LOG.info(result.getException().getMessage()); Throwable cause = ProcedureTestingUtility.getExceptionCause(result); - assertTrue("expected TestProcedureException, got " + cause, - cause instanceof TestProcedureException); + assertTrue(cause instanceof TestProcedureException, + "expected TestProcedureException, got " + cause); - assertEquals(state.toString(), 6, state.size()); + assertEquals(6, state.size(), state.toString()); assertEquals("rootProc-execute", state.get(0)); assertEquals("subProc1-execute", state.get(1)); assertEquals("subProc2-execute", state.get(2)); @@ -216,10 +213,10 @@ protected Procedure[] execute(Void env) { @Override protected void rollback(Void env) throws IOException { if (++retries < 3) { - LOG.info("inject rollback failure " + retries); + LOG.info("inject rollback failure {}", retries); throw new IOException("injected failure number " + retries); } - LOG.info("execute non faulty rollback step retries=" + retries); + LOG.info("execute non faulty rollback step retries={}", retries); } @Override @@ -233,11 +230,11 @@ public void testRollbackRetriableFailure() { long procId = ProcedureTestingUtility.submitAndWait(procExecutor, new TestFaultyRollback()); Procedure result = procExecutor.getResult(procId); - assertTrue("expected a failure", result.isFailed()); + assertTrue(result.isFailed(), "expected a failure"); LOG.info(result.getException().getMessage()); Throwable cause = ProcedureTestingUtility.getExceptionCause(result); - assertTrue("expected TestProcedureException, got " + cause, - cause instanceof TestProcedureException); + assertInstanceOf(TestProcedureException.class, cause, + "expected TestProcedureException, got " + cause); } public static class TestWaitingProcedure extends SequentialProcedure { @@ -315,11 +312,11 @@ public void testAbortTimeout() { long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); long execTime = EnvironmentEdgeManager.currentTime() - startTime; LOG.info(Objects.toString(state)); - assertTrue("we didn't wait enough execTime=" + execTime, execTime >= PROC_TIMEOUT_MSEC); + assertTrue(execTime >= PROC_TIMEOUT_MSEC, "we didn't wait enough execTime=" + execTime); Procedure result = procExecutor.getResult(rootId); - assertTrue(state.toString(), result.isFailed()); + assertTrue(result.isFailed(), state.toString()); ProcedureTestingUtility.assertIsTimeoutException(result); - assertEquals(state.toString(), 2, state.size()); + assertEquals(2, state.size(), state.toString()); assertEquals("wproc-execute", state.get(0)); assertEquals("wproc-rollback", state.get(1)); } @@ -332,9 +329,9 @@ public void testAbortTimeoutWithChildren() { long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); LOG.info(Objects.toString(state)); Procedure result = procExecutor.getResult(rootId); - assertTrue(state.toString(), result.isFailed()); + assertTrue(result.isFailed(), state.toString()); ProcedureTestingUtility.assertIsTimeoutException(result); - assertEquals(state.toString(), 3, state.size()); + assertEquals(3, state.size(), state.toString()); assertEquals("wproc-execute", state.get(0)); assertEquals("wproc-child-execute", state.get(1)); assertEquals("wproc-rollback", state.get(2)); diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java index 41f521cd29aa..5959ca524b3f 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java @@ -17,33 +17,30 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; import org.apache.hadoop.hbase.procedure2.store.NoopProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Threads; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureExecutor { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureExecutor.class); - private static final Logger LOG = LoggerFactory.getLogger(TestProcedureExecutor.class); private TestProcEnv procEnv; @@ -52,7 +49,7 @@ public class TestProcedureExecutor { private HBaseCommonTestingUtility htu; - @Before + @BeforeEach public void setUp() throws Exception { htu = new HBaseCommonTestingUtility(); @@ -62,7 +59,7 @@ public void setUp() throws Exception { procStore.start(1); } - @After + @AfterEach public void tearDown() throws Exception { procExecutor.stop(); procStore.stop(false); @@ -106,9 +103,9 @@ public void testWorkerStuck() throws Exception { assertEquals(true, procExecutor.isFinished(otherProcId)); ProcedureTestingUtility.assertProcNotFailed(procExecutor, otherProcId); - assertEquals(true, procExecutor.isRunning()); - assertEquals(false, procExecutor.isFinished(busyProcId1)); - assertEquals(false, procExecutor.isFinished(busyProcId2)); + assertTrue(procExecutor.isRunning()); + assertFalse(procExecutor.isFinished(busyProcId1)); + assertFalse(procExecutor.isFinished(busyProcId2)); // terminate the busy procedures latch1.release(); @@ -117,7 +114,7 @@ public void testWorkerStuck() throws Exception { LOG.info("set keep alive and wait threads being removed"); procExecutor.setKeepAliveTime(500L, TimeUnit.MILLISECONDS); int threads2 = waitThreadCount(NUM_THREADS); - LOG.info("threads got removed: " + (threads1 - threads2)); + LOG.info("threads got removed: {}", threads1 - threads2); assertEquals(NUM_THREADS, threads2); // terminate the busy procedures diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java index 9e83cc2e9e42..4a1964941549 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java @@ -17,32 +17,27 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.NoopProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureInMemoryChore { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureInMemoryChore.class); - private static final Logger LOG = LoggerFactory.getLogger(TestProcedureInMemoryChore.class); private static final int PROCEDURE_EXECUTOR_SLOTS = 1; @@ -53,7 +48,7 @@ public class TestProcedureInMemoryChore { private HBaseCommonTestingUtility htu; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); @@ -65,7 +60,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -89,9 +84,9 @@ public void testChoreAddAndRemove() throws Exception { latch = new CountDownLatch(nCountDown); chore.setLatch(latch); latch.await(timeoutMSec * nCountDown, TimeUnit.MILLISECONDS); - LOG.info("chore latch count=" + latch.getCount()); + LOG.info("chore latch count={}", latch.getCount()); assertFalse(chore.isWaiting()); - assertTrue("latchCount=" + latch.getCount(), latch.getCount() > 0); + assertTrue(latch.getCount() > 0, "latchCount=" + latch.getCount()); } public static class TestLatchChore extends ProcedureInMemoryChore { @@ -108,7 +103,7 @@ public void setLatch(final CountDownLatch latch) { @Override protected void periodicExecute(final TestProcEnv env) { - LOG.info("periodic execute " + this); + LOG.info("periodic execute {}", this); latch.countDown(); } } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureMetrics.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureMetrics.java index 345ee520b5fc..a50ca7e04cf9 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureMetrics.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureMetrics.java @@ -17,33 +17,25 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Category({ MasterTests.class, SmallTests.class }) -public class TestProcedureMetrics { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureMetrics.class); +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; - private static final Logger LOG = LoggerFactory.getLogger(TestProcedureMetrics.class); +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) +public class TestProcedureMetrics { private static final int PROCEDURE_EXECUTOR_SLOTS = 1; @@ -60,7 +52,7 @@ public class TestProcedureMetrics { private static int successCount = 0; private static int failedCount = 0; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); @@ -76,7 +68,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -88,13 +80,13 @@ public void testMetricForSimpleProcedure() throws Exception { // procedure that executes successfully ProcedureMetrics proc = new ProcedureMetrics(true); long id = ProcedureTestingUtility.submitAndWait(procExecutor, proc); - assertNotEquals("ProcId zero!", 0, id); + assertNotEquals(0, id, "ProcId zero!"); beginCount++; successCount++; ProcedureTestingUtility.waitProcedure(procExecutor, proc); - assertEquals("beginCount doesn't match!", beginCount, proc.beginCount); - assertEquals("successCount doesn't match!", successCount, proc.successCount); - assertEquals("failedCont doesn't match!", failedCount, proc.failedCount); + assertEquals(beginCount, proc.beginCount, "beginCount doesn't match!"); + assertEquals(successCount, proc.successCount, "successCount doesn't match!"); + assertEquals(failedCount, proc.failedCount, "failedCont doesn't match!"); } @Test @@ -102,13 +94,13 @@ public void testMetricsForFailedProcedure() throws Exception { // procedure that fails ProcedureMetrics proc = new ProcedureMetrics(false); long id = ProcedureTestingUtility.submitAndWait(procExecutor, proc); - assertNotEquals("ProcId zero!", 0, id); + assertNotEquals(0, id, "ProcId zero!"); beginCount++; failedCount++; ProcedureTestingUtility.waitProcedure(procExecutor, proc); - assertEquals("beginCount doesn't match!", beginCount, proc.beginCount); - assertEquals("successCount doesn't match!", successCount, proc.successCount); - assertEquals("failedCont doesn't match!", failedCount, proc.failedCount); + assertEquals(beginCount, proc.beginCount, "beginCount doesn't match!"); + assertEquals(successCount, proc.successCount, "successCount doesn't match!"); + assertEquals(failedCount, proc.failedCount, "failedCont doesn't match!"); } @Test @@ -116,13 +108,13 @@ public void testMetricForYieldProcedure() throws Exception { // procedure that yields ProcedureMetrics proc = new ProcedureMetrics(true, true); long id = ProcedureTestingUtility.submitAndWait(procExecutor, proc); - assertNotEquals("ProcId zero!", 0, id); + assertNotEquals(0, id, "ProcId zero!"); beginCount++; successCount++; ProcedureTestingUtility.waitProcedure(procExecutor, proc); - assertEquals("beginCount doesn't match!", beginCount, proc.beginCount); - assertEquals("successCount doesn't match!", successCount, proc.successCount); - assertEquals("failedCont doesn't match!", failedCount, proc.failedCount); + assertEquals(beginCount, proc.beginCount, "beginCount doesn't match!"); + assertEquals(successCount, proc.successCount, "successCount doesn't match!"); + assertEquals(failedCount, proc.failedCount, "failedCont doesn't match!"); } @Test @@ -130,13 +122,13 @@ public void testMetricForFailedYiledProcedure() { // procedure that yields and fails ProcedureMetrics proc = new ProcedureMetrics(false, true); long id = ProcedureTestingUtility.submitAndWait(procExecutor, proc); - assertNotEquals("ProcId zero!", 0, id); + assertNotEquals(0, id, "ProcId zero!"); beginCount++; failedCount++; ProcedureTestingUtility.waitProcedure(procExecutor, proc); - assertEquals("beginCount doesn't match!", beginCount, proc.beginCount); - assertEquals("successCount doesn't match!", successCount, proc.successCount); - assertEquals("failedCont doesn't match!", failedCount, proc.failedCount); + assertEquals(beginCount, proc.beginCount, "beginCount doesn't match!"); + assertEquals(successCount, proc.successCount, "successCount doesn't match!"); + assertEquals(failedCount, proc.failedCount, "failedCont doesn't match!"); } @Test @@ -152,7 +144,7 @@ public void testMetricForProcedureWithChildren() throws Exception { ProcedureMetrics proc = new ProcedureMetrics(true, true, 3, subprocs); long id = ProcedureTestingUtility.submitAndWait(procExecutor, proc); - assertNotEquals("ProcId zero!", 0, id); + assertNotEquals(0, id, "ProcId zero!"); beginCount += subProcCount + 1; successCount += subProcCount - (failChildIndex + 1); if (failChildIndex >= 0) { @@ -161,9 +153,9 @@ public void testMetricForProcedureWithChildren() throws Exception { successCount++; } ProcedureTestingUtility.waitProcedure(procExecutor, proc); - assertEquals("beginCount doesn't match!", beginCount, proc.beginCount); - assertEquals("successCount doesn't match!", successCount, proc.successCount); - assertEquals("failedCont doesn't match!", failedCount, proc.failedCount); + assertEquals(beginCount, proc.beginCount, "beginCount doesn't match!"); + assertEquals(successCount, proc.successCount, "successCount doesn't match!"); + assertEquals(failedCount, proc.failedCount, "failedCont doesn't match!"); } private static class TestProcEnv { diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureNonce.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureNonce.java index 6ebc51471a79..dd95a97ccf5f 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureNonce.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureNonce.java @@ -17,16 +17,16 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.security.User; @@ -35,19 +35,16 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.NonceKey; import org.apache.hadoop.hbase.util.Threads; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureNonce { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureNonce.class); private static final Logger LOG = LoggerFactory.getLogger(TestProcedureNonce.class); @@ -61,7 +58,7 @@ public class TestProcedureNonce { private FileSystem fs; private Path logDir; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); Path testDir = htu.getDataTestDir(); @@ -77,7 +74,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -190,8 +187,8 @@ private void testConcurrentNonceRegistration(final boolean submitProcedure, fina public void run() { try { // release the nonce and wake t2 - assertFalse("unexpected already registered nonce", - procExecutor.registerNonce(nonceKey) >= 0); + assertFalse(procExecutor.registerNonce(nonceKey) >= 0, + "unexpected already registered nonce"); t1NonceRegisteredLatch.countDown(); // hold the submission until t2 is registering the nonce @@ -229,7 +226,7 @@ public void run() { // register the nonce t2BeforeNonceRegisteredLatch.countDown(); - assertFalse("unexpected non registered nonce", procExecutor.registerNonce(nonceKey) < 0); + assertFalse(procExecutor.registerNonce(nonceKey) < 0, "unexpected non registered nonce"); } catch (Throwable e) { t2Exception.set(e); } finally { @@ -248,8 +245,8 @@ public void run() { } ProcedureTestingUtility.waitNoProcedureRunning(procExecutor); - assertEquals(null, t1Exception.get()); - assertEquals(null, t2Exception.get()); + assertNull(t1Exception.get()); + assertNull(t2Exception.get()); } public static class TestSingleStepProcedure extends SequentialProcedure { @@ -262,7 +259,7 @@ public TestSingleStepProcedure() { protected Procedure[] execute(TestProcEnv env) throws InterruptedException { step++; env.waitOnLatch(); - LOG.debug("execute procedure " + this + " step=" + step); + LOG.debug("execute procedure {} step={}", this, step); step++; setResult(Bytes.toBytes(step)); return null; diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java index 8fd02e412d86..526a90cbbaca 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.concurrent.CountDownLatch; @@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -35,21 +34,18 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Threads; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.protobuf.Int32Value; -@Category({ MasterTests.class, MediumTests.class }) +@Tag(MasterTests.TAG) +@Tag(MediumTests.TAG) public class TestProcedureRecovery { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureRecovery.class); private static final Logger LOG = LoggerFactory.getLogger(TestProcedureRecovery.class); @@ -65,7 +61,7 @@ public class TestProcedureRecovery { private Path testDir; private Path logDir; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); @@ -82,7 +78,7 @@ public void setUp() throws IOException { procSleepInterval = 0; } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -531,7 +527,7 @@ private void dumpLogDirState() { FileStatus[] files = fs.listStatus(logDir); if (files != null && files.length > 0) { for (FileStatus file : files) { - assertTrue(file.toString(), file.isFile()); + assertTrue(file.isFile(), file.toString()); LOG.debug("log file " + file.getPath() + " size=" + file.getLen()); } } else { diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java index c2564eabd729..f54051cf0e8e 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java @@ -17,27 +17,25 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.util.ArrayList; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,13 +45,10 @@ * For now we do not guarantee this, we will restore the locks when restarting ProcedureExecutor so * we should use lock to obtain the correct order. Ignored. */ -@Ignore -@Category({ MasterTests.class, SmallTests.class }) +@Disabled +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureReplayOrder { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureReplayOrder.class); - private static final Logger LOG = LoggerFactory.getLogger(TestProcedureReplayOrder.class); private static final int NUM_THREADS = 16; @@ -67,7 +62,7 @@ public class TestProcedureReplayOrder { private Path testDir; private Path logDir; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); htu.getConfiguration().setInt(WALProcedureStore.SYNC_WAIT_MSEC_CONF_KEY, 25); @@ -84,7 +79,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, 1, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -184,7 +179,7 @@ public void assertSortedExecList(int numProcs) { for (int i = 0; i < execList.size() - 1; ++i) { TestProcedure a = execList.get(i); TestProcedure b = execList.get(i + 1); - assertTrue("exec list not sorted: " + a + " < " + b, a.getExecId() > b.getExecId()); + assertTrue(a.getExecId() > b.getExecId(), "exec list not sorted: " + a + " < " + b); } } } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRollbackAIOOB.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRollbackAIOOB.java index 21d8efb1d15a..de2d38599e9a 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRollbackAIOOB.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRollbackAIOOB.java @@ -19,29 +19,24 @@ import java.io.IOException; import java.util.concurrent.CountDownLatch; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; /** * Testcase for HBASE-20973 */ -@Category({ MasterTests.class, MediumTests.class }) +@Tag(MasterTests.TAG) +@Tag(MediumTests.TAG) public class TestProcedureRollbackAIOOB { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureRollbackAIOOB.class); private static final HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility(); @@ -77,25 +72,22 @@ protected Procedure[] execute(Void env) private ProcedureExecutor procExec; - @Rule - public final TestName name = new TestName(); - - @Before - public void setUp() throws IOException { + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException { procStore = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), - UTIL.getDataTestDir(name.getMethodName())); + UTIL.getDataTestDir(testInfo.getTestMethod().get().getName())); procStore.start(2); - procExec = new ProcedureExecutor(UTIL.getConfiguration(), null, procStore); + procExec = new ProcedureExecutor<>(UTIL.getConfiguration(), null, procStore); ProcedureTestingUtility.initAndStartWorkers(procExec, 2, true); } - @After + @AfterEach public void tearDown() { procExec.stop(); procStore.stop(false); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws IOException { UTIL.cleanupTestDir(); } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSchedulerConcurrency.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSchedulerConcurrency.java index 186d38595344..9ab882ee0efa 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSchedulerConcurrency.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSchedulerConcurrency.java @@ -20,37 +20,34 @@ import java.io.IOException; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Threads; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureSchedulerConcurrency { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureSchedulerConcurrency.class); - private static final Logger LOG = LoggerFactory.getLogger(TestProcedureEvents.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestProcedureSchedulerConcurrency.class); private SimpleProcedureScheduler procSched; - @Before + @BeforeEach public void setUp() throws IOException { procSched = new SimpleProcedureScheduler(); procSched.start(); } - @After + @AfterEach public void tearDown() throws IOException { procSched.stop(); } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSkipPersistence.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSkipPersistence.java index 108acd11af6a..1119872e8412 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSkipPersistence.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSkipPersistence.java @@ -17,31 +17,27 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureSkipPersistence { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureSkipPersistence.class); private ProcedureExecutor procExecutor; private ProcedureStore procStore; @@ -123,7 +119,7 @@ protected void deserializeStateData(ProcedureStateSerializer serializer) throws } } - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); @@ -137,7 +133,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, 1, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSuspended.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSuspended.java index 94c48647635f..52a2d39a6f1c 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSuspended.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureSuspended.java @@ -17,32 +17,30 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.NoopProcedureStore; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Threads; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureSuspended { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureSuspended.class); private static final Logger LOG = LoggerFactory.getLogger(TestProcedureSuspended.class); @@ -54,7 +52,7 @@ public class TestProcedureSuspended { private HBaseCommonTestingUtility htu; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); @@ -64,7 +62,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -87,8 +85,8 @@ public void testSuspendWhileHoldingLocks() { waitAndAssertTimestamp(p1keyA, 1, 1); waitAndAssertTimestamp(p2keyA, 0, -1); waitAndAssertTimestamp(p3keyB, 1, 2); - assertEquals(true, lockA.get()); - assertEquals(true, lockB.get()); + assertTrue(lockA.get()); + assertTrue(lockB.get()); // release p3 p3keyB.setThrowSuspend(false); @@ -96,11 +94,11 @@ public void testSuspendWhileHoldingLocks() { waitAndAssertTimestamp(p1keyA, 1, 1); waitAndAssertTimestamp(p2keyA, 0, -1); waitAndAssertTimestamp(p3keyB, 2, 3); - assertEquals(true, lockA.get()); + assertTrue(lockA.get()); // wait until p3 is fully completed ProcedureTestingUtility.waitProcedure(procExecutor, p3keyB); - assertEquals(false, lockB.get()); + assertFalse(lockB.get()); // rollback p2 and wait until is fully completed p1keyA.setTriggerRollback(true); @@ -111,7 +109,7 @@ public void testSuspendWhileHoldingLocks() { waitAndAssertTimestamp(p1keyA, 4, 60000); waitAndAssertTimestamp(p2keyA, 1, 7); waitAndAssertTimestamp(p3keyB, 2, 3); - assertEquals(true, lockA.get()); + assertTrue(lockA.get()); // wait until p2 is fully completed p2keyA.setThrowSuspend(false); @@ -120,8 +118,8 @@ public void testSuspendWhileHoldingLocks() { waitAndAssertTimestamp(p1keyA, 4, 60000); waitAndAssertTimestamp(p2keyA, 2, 8); waitAndAssertTimestamp(p3keyB, 2, 3); - assertEquals(false, lockA.get()); - assertEquals(false, lockB.get()); + assertFalse(lockA.get()); + assertFalse(lockB.get()); } @Test @@ -164,7 +162,7 @@ private void waitAndAssertTimestamp(TestLockProcedure proc, int size, int lastTs Threads.sleep(10); } - LOG.info(proc + " -> " + timestamps); + LOG.info("{} -> {}", proc, timestamps); assertEquals(size, timestamps.size()); if (size > 0) { assertEquals(lastTs, timestamps.get(timestamps.size() - 1).longValue()); @@ -204,7 +202,7 @@ public void setTriggerRollback(final boolean triggerRollback) { @Override protected Procedure[] execute(final TestProcEnv env) throws ProcedureYieldException, ProcedureSuspendedException { - LOG.info("EXECUTE " + this + " suspend " + (lock != null)); + LOG.info("EXECUTE {} suspend {}", this, lock != null); timestamps.add(env.nextTimestamp()); if (triggerRollback) { setFailure(getClass().getSimpleName(), new Exception("injected failure")); @@ -218,7 +216,7 @@ protected Procedure[] execute(final TestProcEnv env) @Override protected void rollback(final TestProcEnv env) { - LOG.info("ROLLBACK " + this); + LOG.info("ROLLBACK {}", this); timestamps.add(env.nextTimestamp() * 10000); } @@ -226,7 +224,7 @@ protected void rollback(final TestProcEnv env) { protected LockState acquireLock(final TestProcEnv env) { hasLock = lock.compareAndSet(false, true); if (hasLock) { - LOG.info("ACQUIRE LOCK " + this + " " + (hasLock)); + LOG.info("ACQUIRE LOCK {} {}", this, hasLock); return LockState.LOCK_ACQUIRED; } return LockState.LOCK_YIELD_WAIT; @@ -234,7 +232,7 @@ protected LockState acquireLock(final TestProcEnv env) { @Override protected void releaseLock(final TestProcEnv env) { - LOG.info("RELEASE LOCK " + this + " " + hasLock); + LOG.info("RELEASE LOCK {} {}", this, hasLock); lock.set(false); } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureToString.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureToString.java index 1ff020d19f1f..54827c8f8331 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureToString.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureToString.java @@ -17,26 +17,21 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ServerCrashState; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureToString { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureToString.class); - /** * A do-nothing environment for BasicProcedure. */ diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureUtil.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureUtil.java index 885ba88a8327..5d35882d4cfc 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureUtil.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureUtil.java @@ -17,33 +17,32 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.TestProcedure; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureUtil.class); - @Test public void testValidation() throws Exception { ProcedureUtil.validateClass(new TestProcedure(10)); } - @Test(expected = BadProcedureException.class) + @Test public void testNoDefaultConstructorValidation() throws Exception { - ProcedureUtil.validateClass(new TestProcedureNoDefaultConstructor(1)); + assertThrows(BadProcedureException.class, () -> { + ProcedureUtil.validateClass(new TestProcedureNoDefaultConstructor(1)); + }); } @Test @@ -53,8 +52,8 @@ public void testConvert() throws Exception { final ProcedureProtos.Procedure proto1 = ProcedureUtil.convertToProtoProcedure(proc1); final TestProcedure proc2 = (TestProcedure) ProcedureUtil.convertToProcedure(proto1); final ProcedureProtos.Procedure proto2 = ProcedureUtil.convertToProtoProcedure(proc2); - assertEquals(false, proto2.hasResult()); - assertEquals("Procedure protobuf does not match", proto1, proto2); + assertFalse(proto2.hasResult()); + assertEquals(proto1, proto2, "Procedure protobuf does not match"); } public static class TestProcedureNoDefaultConstructor extends TestProcedure { diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestRemoteProcedureDispatcherUncaughtExceptionHandler.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestRemoteProcedureDispatcherUncaughtExceptionHandler.java index 3bf6c4741145..4cb9bc64a26c 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestRemoteProcedureDispatcherUncaughtExceptionHandler.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestRemoteProcedureDispatcherUncaughtExceptionHandler.java @@ -17,20 +17,19 @@ */ package org.apache.hadoop.hbase.procedure2; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.lang.Thread.UncaughtExceptionHandler; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Make sure the {@link UncaughtExceptionHandler} will be called when there are unchecked exceptions @@ -38,15 +37,12 @@ *

* See HBASE-21875 and HBASE-21890 for more details. */ -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestRemoteProcedureDispatcherUncaughtExceptionHandler { private static HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility(); - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRemoteProcedureDispatcherUncaughtExceptionHandler.class); - private static final class ExceptionHandler implements UncaughtExceptionHandler { private Throwable error; @@ -88,21 +84,18 @@ protected void abortPendingOperations(Integer key, Set operatio } } - @Rule - public ExpectedException thrown = ExpectedException.none(); - private ExceptionHandler handler; private Dispatcher dispatcher; - @Before + @BeforeEach public void setUp() { handler = new ExceptionHandler(); dispatcher = new Dispatcher(handler); dispatcher.start(); } - @After + @AfterEach public void tearDown() { dispatcher.stop(); dispatcher = null; @@ -112,8 +105,6 @@ public void tearDown() { @Test public void testSubmit() throws Throwable { String message = "inject error"; - thrown.expect(RuntimeException.class); - thrown.expectMessage(message); dispatcher.submitTask(new Runnable() { @Override @@ -121,14 +112,13 @@ public void run() { throw new RuntimeException(message); } }); - handler.get(); + RuntimeException exception = assertThrows(RuntimeException.class, () -> handler.get()); + assertEquals(message, exception.getMessage()); } @Test public void testDelayedSubmit() throws Throwable { String message = "inject error"; - thrown.expect(RuntimeException.class); - thrown.expectMessage(message); dispatcher.submitTask(new Runnable() { @Override @@ -136,6 +126,7 @@ public void run() { throw new RuntimeException(message); } }, 100, TimeUnit.MILLISECONDS); - handler.get(); + RuntimeException exception = assertThrows(RuntimeException.class, () -> handler.get()); + assertEquals(message, exception.getMessage()); } } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java index b4addae22d81..e4a978b4a0f1 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStackIdHoles.java @@ -22,7 +22,6 @@ import java.util.LinkedHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase; @@ -30,11 +29,10 @@ import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.AtomicUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; @@ -43,13 +41,10 @@ * {@link RootProcedureState} first and then crash, and then cause holes in stack ids when loading, * and finally fail the start up of master. */ -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestStackIdHoles { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStackIdHoles.class); - private final class DummyProcedureStore extends ProcedureStoreBase { private int numThreads; @@ -193,7 +188,7 @@ public void delete(long[] procIds, int offset, int count) { private ProcedureExecutor procExec; - @Before + @BeforeEach public void setUp() throws IOException { procStore = new DummyProcedureStore(); procStore.start(4); @@ -202,7 +197,7 @@ public void setUp() throws IOException { procExec.startWorkers(); } - @After + @AfterEach public void tearDown() { procExec.stop(); } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java index 831ff59aac19..f07cb6ec89d3 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java @@ -17,32 +17,28 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, MediumTests.class }) +@Tag(MasterTests.TAG) +@Tag(MediumTests.TAG) public class TestStateMachineProcedure { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStateMachineProcedure.class); private static final Logger LOG = LoggerFactory.getLogger(TestStateMachineProcedure.class); @@ -81,7 +77,7 @@ public int hashCode() { private Path testDir; private Path logDir; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); @@ -94,10 +90,10 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } - @After + @AfterEach public void tearDown() throws IOException { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExecutor, false); - assertTrue("expected executor to be running", procExecutor.isRunning()); + assertTrue(procExecutor.isRunning(), "expected executor to be running"); procExecutor.stop(); procStore.stop(false); @@ -113,7 +109,7 @@ public void testAbortStuckProcedure() throws InterruptedException { Thread.sleep(1000 + (int) (Math.random() * 4001)); proc.abort(procExecutor.getEnvironment()); ProcedureTestingUtility.waitProcedure(procExecutor, procId); - assertEquals(true, proc.isFailed()); + assertTrue(proc.isFailed()); } finally { procExecutor.getEnvironment().loop = false; } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java index c43139e3e170..cefe12da413b 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hbase.procedure2; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -27,24 +28,20 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestYieldProcedures { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestYieldProcedures.class); private static final Logger LOG = LoggerFactory.getLogger(TestYieldProcedures.class); @@ -60,7 +57,7 @@ public class TestYieldProcedures { private Path testDir; private Path logDir; - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); @@ -76,7 +73,7 @@ public void setUp() throws IOException { ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); } - @After + @AfterEach public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); @@ -101,14 +98,14 @@ public void testYieldEachExecutionStep() throws Exception { int index = 0; for (int execStep = 0; execStep < NUM_STATES; ++execStep) { TestStateMachineProcedure.ExecutionInfo info = procs[i].getExecutionInfo().get(index++); - assertEquals(false, info.isRollback()); + assertFalse(info.isRollback()); assertEquals(execStep, info.getStep().ordinal()); } // verify rollback for (int execStep = NUM_STATES - 1; execStep >= 0; --execStep) { TestStateMachineProcedure.ExecutionInfo info = procs[i].getExecutionInfo().get(index++); - assertEquals(true, info.isRollback()); + assertTrue(info.isRollback()); assertEquals(execStep, info.getStep().ordinal()); } } @@ -134,18 +131,18 @@ public void testYieldOnInterrupt() throws Exception { assertEquals(NUM_STATES * 4, proc.getExecutionInfo().size()); for (int i = 0; i < NUM_STATES; ++i) { TestStateMachineProcedure.ExecutionInfo info = proc.getExecutionInfo().get(count++); - assertEquals(false, info.isRollback()); + assertFalse(info.isRollback()); assertEquals(i, info.getStep().ordinal()); info = proc.getExecutionInfo().get(count++); - assertEquals(false, info.isRollback()); + assertFalse(info.isRollback()); assertEquals(i, info.getStep().ordinal()); } // test rollback (we execute steps twice, rollback counts both IE and completed) for (int i = NUM_STATES - 1; i >= 0; --i) { TestStateMachineProcedure.ExecutionInfo info = proc.getExecutionInfo().get(count++); - assertEquals(true, info.isRollback()); + assertTrue(info.isRollback()); assertEquals(i, info.getStep().ordinal()); } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureTree.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureTree.java index 73acdf633ce6..6799bcc98ada 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureTree.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureTree.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.procedure2.store; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.io.UncheckedIOException; @@ -25,7 +25,6 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; @@ -34,19 +33,15 @@ import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestProcedureTree { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureTree.class); - public static final class TestProcedure extends Procedure { @Override @@ -169,5 +164,4 @@ public void testOrphan() throws IOException { assertEquals(5, corruptedProcs.get(0).getProcId()); assertEquals(4, corruptedProcs.get(0).getParentProcId()); } - } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestBitSetNode.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestBitSetNode.java index 251cb39842b2..762a958aa101 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestBitSetNode.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestBitSetNode.java @@ -17,26 +17,21 @@ */ package org.apache.hadoop.hbase.procedure2.store.wal; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.store.wal.ProcedureStoreTracker.DeleteState; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestBitSetNode { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestBitSetNode.class); - @Test public void testGetActiveMaxMinProcId() { BitSetNode node = new BitSetNode(5L, false); diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestProcedureStoreTracker.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestProcedureStoreTracker.java index 46164be64698..593f605967ea 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestProcedureStoreTracker.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestProcedureStoreTracker.java @@ -17,26 +17,23 @@ */ package org.apache.hadoop.hbase.procedure2.store.wal; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.ThreadLocalRandom; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, MediumTests.class }) +@Tag(MasterTests.TAG) +@Tag(MediumTests.TAG) public class TestProcedureStoreTracker { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestProcedureStoreTracker.class); private static final Logger LOG = LoggerFactory.getLogger(TestProcedureStoreTracker.class); @@ -58,7 +55,7 @@ public void testSeqInsertAndDelete() { } // All the proc that are not yet inserted should be result as deleted for (int j = i + 1; j < MAX_PROC; ++j) { - assertTrue(tracker.isDeleted(j) != ProcedureStoreTracker.DeleteState.NO); + assertNotSame(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j)); } } @@ -213,7 +210,7 @@ public void testDelete() { if (hasProc) { assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(i)); } else { - assertEquals("procId=" + i, ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(i)); + assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(i), "procId=" + i); } } } @@ -227,24 +224,24 @@ public void testSetDeletedIfModified() { for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); } - assertEquals(false, tracker.isEmpty()); + assertFalse(tracker.isEmpty()); for (int i = 0; i < procIds.length; ++i) { tracker.setDeletedIfModified(procIds[i] - 1); tracker.setDeletedIfModified(procIds[i]); tracker.setDeletedIfModified(procIds[i] + 1); } - assertEquals(true, tracker.isEmpty()); + assertTrue(tracker.isEmpty()); // test batch tracker.reset(); for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); } - assertEquals(false, tracker.isEmpty()); + assertFalse(tracker.isEmpty()); tracker.setDeletedIfModified(procIds); - assertEquals(true, tracker.isEmpty()); + assertTrue(tracker.isEmpty()); } @Test diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestStressWALProcedureStore.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestStressWALProcedureStore.java index 4bdd3f268a1b..5655555c1f12 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestStressWALProcedureStore.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestStressWALProcedureStore.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.procedure2.store.wal; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Random; @@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter; @@ -35,23 +34,19 @@ import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, MediumTests.class }) +@Tag(MasterTests.TAG) +@Tag(MediumTests.TAG) public class TestStressWALProcedureStore { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStressWALProcedureStore.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestWALProcedureStore.class); + private static final Logger LOG = LoggerFactory.getLogger(TestStressWALProcedureStore.class); private static final int PROCEDURE_STORE_SLOTS = 8; @@ -68,7 +63,7 @@ private void setupConfiguration(Configuration conf) { conf.setInt(WALProcedureStore.ROLL_THRESHOLD_CONF_KEY, 128 * 1024); } - @Before + @BeforeEach public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); setupConfiguration(htu.getConfiguration()); @@ -89,7 +84,7 @@ public void setUp() throws IOException { assertEquals(0, loader.getCorruptedCount()); } - @After + @AfterEach public void tearDown() throws IOException { procStore.stop(false); fs.delete(logDir, true); @@ -138,14 +133,14 @@ public void run() { assertEquals(1, procStore.getActiveLogs().size()); } - @Ignore + @Disabled @Test // REENABLE after merge of // https://github.com/google/protobuf/issues/2228#issuecomment-252058282 public void testEntrySizeLimit() throws Exception { final int NITEMS = 20; for (int i = 1; i <= NITEMS; ++i) { final byte[] data = new byte[256 << i]; - LOG.info(String.format("Writing %s", StringUtils.humanSize(data.length))); + LOG.info("Writing {}", StringUtils.humanSize(data.length)); TestProcedure proc = new TestProcedure(i, 0, data); procStore.insert(proc, null); } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java index 1df22ae75338..9c6a1a7e42cf 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java @@ -17,9 +17,10 @@ */ package org.apache.hadoop.hbase.procedure2.store.wal; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.FileNotFoundException; import java.io.IOException; @@ -35,7 +36,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.procedure2.Procedure; @@ -49,11 +49,10 @@ import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -62,11 +61,9 @@ import org.apache.hbase.thirdparty.com.google.protobuf.Int64Value; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestWALProcedureStore { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestWALProcedureStore.class); private static final Logger LOG = LoggerFactory.getLogger(TestWALProcedureStore.class); @@ -83,7 +80,7 @@ private void setupConfig(final Configuration conf) { conf.setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, true); } - @Before + @BeforeEach public void setUp() throws IOException { testDir = htu.getDataTestDir(); htu.getConfiguration().set(HConstants.HBASE_DIR, testDir.toString()); @@ -100,7 +97,7 @@ public void setUp() throws IOException { procStore.load(new LoadCounter()); } - @After + @AfterEach public void tearDown() throws IOException { procStore.stop(false); fs.delete(logDir, true); @@ -171,8 +168,8 @@ public void testWalCleanerSequentialClean() throws Exception { for (int i = 0; i < deleteOrder.length; i++) { procStore.delete(procs[deleteOrder[i]].getProcId()); procStore.removeInactiveLogsForTesting(); - assertFalse(logs.get(deleteOrder[i]).toString(), - procStore.getActiveLogs().contains(logs.get(deleteOrder[i]))); + assertFalse(procStore.getActiveLogs().contains(logs.get(deleteOrder[i])), + logs.get(deleteOrder[i]).toString()); assertEquals(procStore.getActiveLogs().size(), procs.length - i); } } @@ -422,11 +419,11 @@ private static void assertUpdated(final ProcedureStoreTracker tracker, final Pro final int[] updatedProcs, final int[] nonUpdatedProcs) { for (int index : updatedProcs) { long procId = procs[index].getProcId(); - assertTrue("Procedure id : " + procId, tracker.isModified(procId)); + assertTrue(tracker.isModified(procId), "Procedure id : " + procId); } for (int index : nonUpdatedProcs) { long procId = procs[index].getProcId(); - assertFalse("Procedure id : " + procId, tracker.isModified(procId)); + assertFalse(tracker.isModified(procId), "Procedure id : " + procId); } } @@ -434,13 +431,13 @@ private static void assertDeleted(final ProcedureStoreTracker tracker, final Pro final int[] deletedProcs, final int[] nonDeletedProcs) { for (int index : deletedProcs) { long procId = procs[index].getProcId(); - assertEquals("Procedure id : " + procId, ProcedureStoreTracker.DeleteState.YES, - tracker.isDeleted(procId)); + assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(procId), + "Procedure id : " + procId); } for (int index : nonDeletedProcs) { long procId = procs[index].getProcId(); - assertEquals("Procedure id : " + procId, ProcedureStoreTracker.DeleteState.NO, - tracker.isDeleted(procId)); + assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(procId), + "Procedure id : " + procId); } } @@ -523,7 +520,7 @@ public void testCorruptedEntries() throws Exception { LoadCounter loader = new LoadCounter(); storeRestart(loader); - assertTrue(procStore.getCorruptedLogs() != null); + assertNotNull(procStore.getCorruptedLogs()); assertEquals(1, procStore.getCorruptedLogs().size()); assertEquals(87, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); @@ -558,7 +555,7 @@ public void testCorruptedProcedures() throws Exception { // the first log was removed, // we have insert-txn and updates in the others so everything is fine FileStatus[] logs = fs.listStatus(logDir); - assertEquals(Arrays.toString(logs), 2, logs.length); + assertEquals(2, logs.length, Arrays.toString(logs)); Arrays.sort(logs, new Comparator() { @Override public int compare(FileStatus o1, FileStatus o2) { @@ -578,9 +575,9 @@ public int compare(FileStatus o1, FileStatus o2) { assertEquals(0, loader.getLoadedCount()); assertEquals(rootProcs.length, loader.getCorruptedCount()); for (Procedure proc : loader.getCorrupted()) { - assertTrue(proc.toString(), proc.getParentProcId() <= rootProcs.length); - assertTrue(proc.toString(), - proc.getProcId() > rootProcs.length && proc.getProcId() <= (rootProcs.length * 2)); + assertTrue(proc.getParentProcId() <= rootProcs.length, proc.toString()); + assertTrue(proc.getProcId() > rootProcs.length && proc.getProcId() <= (rootProcs.length * 2), + proc.toString()); } } @@ -789,7 +786,7 @@ public void testBatchInsert() throws Exception { restartAndAssert(procId != count ? count : 0, count - (i + 1), 0, 0); } procStore.removeInactiveLogsForTesting(); - assertEquals("WALs=" + procStore.getActiveLogs(), 1, procStore.getActiveLogs().size()); + assertEquals(1, procStore.getActiveLogs().size(), "WALs=" + procStore.getActiveLogs()); } @Test diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java index 27f280ad13ac..f07674d428ae 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java @@ -17,24 +17,19 @@ */ package org.apache.hadoop.hbase.procedure2.util; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ MasterTests.class, SmallTests.class }) +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) public class TestDelayedUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestDelayedUtil.class); - private static final Logger LOG = LoggerFactory.getLogger(TestDelayedUtil.class); @Test @@ -74,8 +69,8 @@ private void assertContainersEquals(final ZeroDelayContainer src, } } boolean isMatching = src.equals(items[i]); - assertEquals(src.getObject() + " unexpectedly match " + items[i].getObject(), shouldMatch, - isMatching); + assertEquals(shouldMatch, isMatching, + src.getObject() + " unexpectedly match " + items[i].getObject()); } } diff --git a/hbase-protocol-shaded/README.txt b/hbase-protocol-shaded/README.txt index b0030fac2157..eb5aaebfcd0d 100644 --- a/hbase-protocol-shaded/README.txt +++ b/hbase-protocol-shaded/README.txt @@ -1,6 +1,9 @@ This module has proto files used by core. These protos overlap with protos that are used by coprocessor endpoints -(CPEP) in the module hbase-protocol. So core versions have -a different name, the generated classes are relocated --- i.e. shaded -- to a new location; they are moved from -org.apache.hadoop.hbase.* to org.apache.hadoop.hbase.shaded. +(CPEP) in the module hbase-protocol. To make sure that the +core versions have a different names, these classes are +using a different package; +they are in +org.apache.hadoop.hbase.shaded.protobuf.generated.* +, while the hbase-protocol classes are in +org.apache.hadoop.hbase.protobuf.generated.*. diff --git a/hbase-protocol-shaded/pom.xml b/hbase-protocol-shaded/pom.xml index b742b35b2fb7..ab2500dfdb6c 100644 --- a/hbase-protocol-shaded/pom.xml +++ b/hbase-protocol-shaded/pom.xml @@ -41,11 +41,6 @@ org.apache.hbase.thirdparty hbase-shaded-protobuf - - junit - junit - test - @@ -58,19 +53,9 @@ maven-surefire-plugin - - - - secondPartTestsExecution - - test - - test - - true - - - + + true + org.xolstice.maven.plugins @@ -93,53 +78,6 @@ com.google.code.maven-replacer-plugin replacer - - org.apache.maven.plugins - maven-shade-plugin - 3.4.1 - - - - shade - - package - - true - true - - - - com.google.protobuf - org.apache.hadoop.hbase.shaded.com.google.protobuf - - - - - - javax.annotation:javax.annotation-api - - org.apache.hbase.thirdparty:* - com.google.protobuf:protobuf-java - com.google.code.findbugs:* - com.google.j2objc:j2objc-annotations - org.codehaus.mojo:animal-sniffer-annotations - junit:junit - commons-logging:commons-logging - org.slf4j:* - org.apache.logging.log4j:* - org.apache.yetus:audience-annotations - com.github.stephenc.fingbugs:* - com.github.spotbugs:* - - - - - - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/hbase-protocol-shaded/src/main/protobuf/Backup.proto b/hbase-protocol-shaded/src/main/protobuf/Backup.proto index a114001ba504..4a830f5d8cbf 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Backup.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Backup.proto @@ -93,6 +93,10 @@ message BackupInfo { optional uint32 workers_number = 11; optional uint64 bandwidth = 12; map table_set_timestamp = 13; + repeated string incr_backup_file_list = 14; + optional uint64 total_bytes_copied = 15; + optional bool no_checksum_verify = 16; + map incr_timestamp_map = 17; message RSTimestampMap { map rs_timestamp = 1; diff --git a/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto b/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto index 80fc10ada786..c1f7b496776f 100644 --- a/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto +++ b/hbase-protocol-shaded/src/main/protobuf/BucketCacheEntry.proto @@ -49,6 +49,9 @@ message BlockCacheKey { required int64 offset = 2; required BlockType block_type = 3; required bool primary_replica_block = 4; + optional string region_name = 5; + optional string family_name = 6; + optional bool archived = 7; } enum BlockType { diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index a70587d34c39..06404e15b771 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -210,6 +210,17 @@ message ModifyTableResponse { optional uint64 proc_id = 1; } +message ReopenTableRegionsRequest { + required TableName table_name = 1; + repeated bytes region_names = 2; // empty = all regions + optional uint64 nonce_group = 3 [default = 0]; + optional uint64 nonce = 4 [default = 0]; +} + +message ReopenTableRegionsResponse { + optional uint64 proc_id = 1; +} + message FlushTableRequest { required TableName table_name = 1; repeated bytes column_family = 2; @@ -900,6 +911,13 @@ service MasterService { rpc ModifyTable(ModifyTableRequest) returns(ModifyTableResponse); + /** + * Reopen regions of a table. Regions are reopened in-place without moving. + * Useful for rolling out table descriptor changes after modifyTable(reopenRegions=false). + */ + rpc ReopenTableRegions(ReopenTableRegionsRequest) + returns(ReopenTableRegionsResponse); + /** Flush a table's memstore */ rpc FlushTable(FlushTableRequest) returns(FlushTableResponse); @@ -1228,6 +1246,9 @@ service MasterService { rpc FlushMasterStore(FlushMasterStoreRequest) returns(FlushMasterStoreResponse); + + rpc RestoreBackupSystemTable(RestoreBackupSystemTableRequest) + returns(RestoreBackupSystemTableResponse); } // HBCK Service definitions. @@ -1313,6 +1334,13 @@ message FixMetaRequest {} message FixMetaResponse {} +message RestoreBackupSystemTableRequest { + required string snapshot_name = 1; +} +message RestoreBackupSystemTableResponse { + optional uint64 proc_id = 1; +} + service HbckService { /** Update state of the table in meta only*/ rpc SetTableStateInMeta(SetTableStateInMetaRequest) diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 024cb7b8b002..ab4daeb39633 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -77,6 +77,7 @@ enum ModifyTableState { MODIFY_TABLE_CLOSE_EXCESS_REPLICAS = 8; MODIFY_TABLE_ASSIGN_NEW_REPLICAS = 9; MODIFY_TABLE_SYNC_ERASURE_CODING_POLICY = 10; + MODIFY_TABLE_SNAPSHOT = 11; } message ModifyTableStateData { @@ -86,6 +87,7 @@ message ModifyTableStateData { required bool delete_column_family_in_modify = 4; optional bool should_check_descriptor = 5; optional bool reopen_regions = 6; + optional string snapshot_name = 7; } enum TruncateTableState { @@ -96,6 +98,7 @@ enum TruncateTableState { TRUNCATE_TABLE_ADD_TO_META = 5; TRUNCATE_TABLE_ASSIGN_REGIONS = 6; TRUNCATE_TABLE_POST_OPERATION = 7; + TRUNCATE_TABLE_SNAPSHOT = 8; } message TruncateTableStateData { @@ -104,6 +107,7 @@ message TruncateTableStateData { optional TableName table_name = 3; optional TableSchema table_schema = 4; repeated RegionInfo region_info = 5; + optional string snapshot_name = 6; } enum TruncateRegionState { @@ -112,6 +116,13 @@ enum TruncateRegionState { TRUNCATE_REGION_REMOVE = 3; TRUNCATE_REGION_MAKE_ONLINE = 4; TRUNCATE_REGION_POST_OPERATION = 5; + TRUNCATE_REGION_SNAPSHOT = 6; +} + +message TruncateRegionStateData { + required UserInformation user_info = 1; + required RegionInfo region_info = 2; + optional string snapshot_name = 3; } enum DeleteTableState { @@ -121,12 +132,14 @@ enum DeleteTableState { DELETE_TABLE_UPDATE_DESC_CACHE = 4; DELETE_TABLE_UNASSIGN_REGIONS = 5; DELETE_TABLE_POST_OPERATION = 6; + DELETE_TABLE_SNAPSHOT = 7; } message DeleteTableStateData { required UserInformation user_info = 1; required TableName table_name = 2; repeated RegionInfo region_info = 3; + optional string snapshot_name = 4; } enum CreateNamespaceState { @@ -730,3 +743,10 @@ message ReloadQuotasProcedureStateData { required ServerName target_server = 1; optional ForeignExceptionMessage error = 2; } + +enum RestoreBackupSystemTableState { + RESTORE_BACKUP_SYSTEM_TABLE_PREPARE = 1; + RESTORE_BACKUP_SYSTEM_TABLE_DISABLE = 2; + RESTORE_BACKUP_SYSTEM_TABLE_RESTORE = 3; + RESTORE_BACKUP_SYSTEM_TABLE_ENABLE = 4; +} diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml index bfa77441d531..ed7b9e7332cd 100644 --- a/hbase-protocol/pom.xml +++ b/hbase-protocol/pom.xml @@ -51,11 +51,6 @@ - - - org.apache.maven.plugins - maven-source-plugin - maven-assembly-plugin @@ -64,20 +59,12 @@ + + org.apache.maven.plugins maven-surefire-plugin - - - - secondPartTestsExecution - - test - - test - - true - - - + + true + org.xolstice.maven.plugins diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml index 531a4bbfb9e2..21c14270c3e9 100644 --- a/hbase-replication/pom.xml +++ b/hbase-replication/pom.xml @@ -100,8 +100,18 @@ zookeeper - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorageTestBase.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorageTestBase.java index e8191d66344c..fbf7c11ba99d 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorageTestBase.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorageTestBase.java @@ -19,13 +19,13 @@ import static org.apache.hadoop.hbase.replication.ReplicationPeerConfigTestUtil.assertConfigEquals; import static org.apache.hadoop.hbase.replication.ReplicationPeerConfigTestUtil.getConfig; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; import java.util.List; -import org.junit.Test; +import org.junit.jupiter.api.Test; public abstract class ReplicationPeerStorageTestBase { diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestCopyReplicationPeers.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestCopyReplicationPeers.java index fdde24852fad..d6d2c53c3869 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestCopyReplicationPeers.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestCopyReplicationPeers.java @@ -19,30 +19,25 @@ import static org.apache.hadoop.hbase.replication.ReplicationPeerConfigTestUtil.assertConfigEquals; import static org.apache.hadoop.hbase.replication.ReplicationPeerConfigTestUtil.getConfig; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseZKTestingUtility; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.util.ToolRunner; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestCopyReplicationPeers { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCopyReplicationPeers.class); - private static final HBaseZKTestingUtility UTIL = new HBaseZKTestingUtility(); private static FileSystem FS; @@ -53,7 +48,7 @@ public class TestCopyReplicationPeers { private static ReplicationPeerStorage DST; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { DIR = UTIL.getDataTestDir("test_peer_migration"); CommonFSUtils.setRootDir(UTIL.getConfiguration(), DIR); @@ -63,7 +58,7 @@ public static void setUp() throws Exception { DST = new FSReplicationPeerStorage(FS, UTIL.getConfiguration()); } - @AfterClass + @AfterAll public static void tearDown() throws Exception { UTIL.shutdownMiniZKCluster(); UTIL.cleanupTestDir(); diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestFSReplicationPeerStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestFSReplicationPeerStorage.java index 13646e661ffa..b7cab8b69cab 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestFSReplicationPeerStorage.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestFSReplicationPeerStorage.java @@ -23,30 +23,25 @@ import java.io.IOException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.util.CommonFSUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestFSReplicationPeerStorage extends ReplicationPeerStorageTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFSReplicationPeerStorage.class); - private static final HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility(); private static FileSystem FS; private static Path DIR; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { DIR = UTIL.getDataTestDir("test_fs_peer_storage"); CommonFSUtils.setRootDir(UTIL.getConfiguration(), DIR); @@ -54,7 +49,7 @@ public static void setUp() throws Exception { STORAGE = new FSReplicationPeerStorage(FS, UTIL.getConfiguration()); } - @AfterClass + @AfterAll public static void tearDown() throws IOException { UTIL.cleanupTestDir(); } diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java index acbcd7aad0f9..2ff1059fd252 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java @@ -19,10 +19,10 @@ import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.util.ArrayList; import java.util.Collections; @@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hbase.zookeeper.ZKConfig; import org.apache.zookeeper.KeeperException; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,8 +87,8 @@ public void testReplicationQueueStorage() throws ReplicationException { List reps = rqs.getListOfReplicators(); assertEquals(2, reps.size()); - assertTrue(server1.getServerName(), reps.contains(server1)); - assertTrue(server2.getServerName(), reps.contains(server2)); + assertTrue(reps.contains(server1), server1.getServerName()); + assertTrue(reps.contains(server2), server2.getServerName()); assertTrue(rqs.getWALsInQueue(ServerName.valueOf("bogus", 12345, 12345), "bogus").isEmpty()); assertTrue(rqs.getWALsInQueue(server1, "bogus").isEmpty()); diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java index 344f8b08463a..e404cc7db866 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java @@ -21,7 +21,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.ClusterId; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseZKTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -32,26 +31,22 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hbase.zookeeper.ZNodePaths; import org.apache.zookeeper.KeeperException; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestReplicationStateZKImpl extends TestReplicationStateBasic { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestReplicationStateZKImpl.class); - private static Configuration conf; private static HBaseZKTestingUtility utility; private static ZKWatcher zkw; private static String replicationZNode; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { utility = new HBaseZKTestingUtility(); utility.startMiniZKCluster(); @@ -77,7 +72,7 @@ private static String initPeerClusterState(String baseZKNode) return ZKConfig.getZooKeeperClusterKey(testConf); } - @Before + @BeforeEach public void setUp() throws IOException { zkTimeoutCount = 0; rqs = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf); @@ -86,12 +81,12 @@ public void setUp() throws IOException { OUR_KEY = ZKConfig.getZooKeeperClusterKey(conf); } - @After + @AfterEach public void tearDown() throws KeeperException, IOException { ZKUtil.deleteNodeRecursively(zkw, replicationZNode); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { utility.shutdownMiniZKCluster(); } diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java index b4c62d5b8ac3..fe49aa31bb42 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java @@ -21,32 +21,27 @@ import static org.hamcrest.MatcherAssert.assertThat; import java.io.IOException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseZKTestingUtility; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.zookeeper.KeeperException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestZKReplicationPeerStorage extends ReplicationPeerStorageTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestZKReplicationPeerStorage.class); - private static final HBaseZKTestingUtility UTIL = new HBaseZKTestingUtility(); - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.startMiniZKCluster(); STORAGE = new ZKReplicationPeerStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration()); } - @AfterClass + @AfterAll public static void tearDown() throws IOException { UTIL.shutdownMiniZKCluster(); } diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java index df8f065dc7c6..19bcd5671686 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java @@ -19,8 +19,8 @@ import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Arrays; @@ -29,7 +29,6 @@ import java.util.Set; import java.util.SortedSet; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseZKTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; @@ -40,38 +39,34 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.zookeeper.KeeperException; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; -@Category({ ReplicationTests.class, MediumTests.class }) +@Tag(ReplicationTests.TAG) +@Tag(MediumTests.TAG) public class TestZKReplicationQueueStorage { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestZKReplicationQueueStorage.class); - private static final HBaseZKTestingUtility UTIL = new HBaseZKTestingUtility(); private static ZKReplicationQueueStorage STORAGE; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { UTIL.startMiniZKCluster(); STORAGE = new ZKReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration()); } - @AfterClass + @AfterAll public static void tearDown() throws IOException { UTIL.shutdownMiniZKCluster(); } - @After + @AfterEach public void tearDownAfterTest() throws ReplicationException, KeeperException, IOException { for (ServerName serverName : STORAGE.getListOfReplicators()) { for (String queue : STORAGE.getAllQueues(serverName)) { diff --git a/hbase-resource-bundle/src/main/resources/META-INF/LICENSE.vm b/hbase-resource-bundle/src/main/resources/META-INF/LICENSE.vm index 15bdb160560b..4ebf986391d0 100644 --- a/hbase-resource-bundle/src/main/resources/META-INF/LICENSE.vm +++ b/hbase-resource-bundle/src/main/resources/META-INF/LICENSE.vm @@ -221,11 +221,11 @@ under the License. limitations under the License. ## Special cases, for e.g. ASL2.0 licensed works that bundle additional third party works -#set($bundled-jquery = ${bundled-jquery.equalsIgnoreCase("true")}) -#set($bundled-vega = ${bundled-vega.equalsIgnoreCase("true")}) -#set($bundled-logo = ${bundled-logo.equalsIgnoreCase("true")}) -#set($bundled-dependencies = ${bundled-dependencies.equalsIgnoreCase("true")}) -#if($bundled-jquery || $bundled-vega || $bundled-logo || $bundled-dependencies) +#set($bundled_jquery_enabled = ${bundled_jquery.equalsIgnoreCase("true")}) +#set($bundled_vega_enabled = ${bundled_vega.equalsIgnoreCase("true")}) +#set($bundled_logo_enabled = ${bundled_logo.equalsIgnoreCase("true")}) +#set($bundled_dependencies_enabled = ${bundled_dependencies.equalsIgnoreCase("true")}) +#if($bundled_jquery_enabled || $bundled_vega_enabled || $bundled_logo_enabled || $bundled_dependencies_enabled) ==== ${project.name} contained works @@ -1341,18 +1341,18 @@ You can redistribute it and/or modify it under either the terms of the PURPOSE. #end ## modules with bundled works in source -#if(${bundled-jquery}) +#if(${bundled_jquery_enabled}) #jquery_license() #end -#if(${bundled-vega}) +#if(${bundled_vega_enabled}) #vega_license() #end -#if(${bundled-logo}) +#if(${bundled_logo_enabled}) #orca_logo_license() #end ## when true, we're in a module that makes a binary dist with ## bundled works. -#if(${bundled-dependencies}) +#if(${bundled_dependencies_enabled}) ==== ## gather up CDDL licensed works #set($cddl_1_0 = []) @@ -1369,7 +1369,7 @@ You can redistribute it and/or modify it under either the terms of the ## gather up EDL works #set($edl_1_0 = []) ## track commons-math -#set($commons-math = false) +#set($commons_math = false) ## track if we need jruby additionals. #set($jruby = false) ## track hadoops @@ -1394,7 +1394,7 @@ g:${dep.groupId} AND a:${dep.artifactId} AND v:${dep.version} Until hbase-resource-bundle/src/main/resources/supplemental-models.xml is updated, the build should fail. #end -#if(${debug-print-included-work-info.equalsIgnoreCase("true")}) +#if(${debug_print_included_work_info.equalsIgnoreCase("true")}) ===== Check license for included work @@ -1417,7 +1417,7 @@ ${dep.scm.url} #set($aggregated=false) ## Check for our set of known dependencies that require manual LICENSE additions. #if($dep.artifactId.equals("commons-math3")) -#set($commons-math=true) +#set($commons_math=true) #end #if($dep.artifactId.equals("jruby-complete")) #set($jruby=true) @@ -1539,7 +1539,7 @@ ${dep.scm.url} #if($hadoop) #hadoop_license() #end -#if($commons-math) +#if($commons_math) #commons_math_license() #end #if(!(${mit.isEmpty()})) diff --git a/hbase-resource-bundle/src/main/resources/META-INF/NOTICE.vm b/hbase-resource-bundle/src/main/resources/META-INF/NOTICE.vm index 8be266c787ef..b1a2bd7bcb93 100644 --- a/hbase-resource-bundle/src/main/resources/META-INF/NOTICE.vm +++ b/hbase-resource-bundle/src/main/resources/META-INF/NOTICE.vm @@ -18,15 +18,15 @@ under the License. *# ## Start with the NOTICE header for this particular artifact ${project.name} -Copyright ${project.inceptionYear}-${copyright-end-year} The Apache Software Foundation +Copyright ${project.inceptionYear}-${copyright_end_year} The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ## Specially handled included deps. e.g. ASL2.0 works that have additional bundled works but -#set($bundled-bootstrap = ${bundled-bootstrap.equalsIgnoreCase("true")}) -#set($bundled-logo = ${bundled-logo.equalsIgnoreCase("true")}) -#set($bundled-dependencies = ${bundled-dependencies.equalsIgnoreCase("true")}) -#if($bundled-bootstrap || $bundled-logo || $bundled-dependencies) +#set($bundled_bootstrap_enabled = ${bundled_bootstrap.equalsIgnoreCase("true")}) +#set($bundled_logo_enabled = ${bundled_logo.equalsIgnoreCase("true")}) +#set($bundled_dependencies_enabled = ${bundled_dependencies.equalsIgnoreCase("true")}) +#if($bundled_bootstrap_enabled || $bundled_logo_enabled || $bundled_dependencies_enabled) ==== ${project.name} contained works @@ -94,13 +94,13 @@ noted in the LICENSE file. #end -#macro(jetty_notice $jetty-include-unixcrypt) +#macro(jetty_notice $jetty_include_unixcrypt) -- This product includes portions of 'The Jetty Web Container' Copyright 1995-2016 Mort Bay Consulting Pty Ltd. -#if(${jetty-include-unixcrypt}) +#if(${jetty_include_unixcrypt}) ## UnixCrypt.java paragraph only in server The UnixCrypt.java code ~Implements the one way cryptography used by Unix systems for simple password protection. Copyright 1996 Aki Yoshida, @@ -199,13 +199,13 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace()) #end ## first bundled source -#if(${bundled-logo}) +#if(${bundled_logo_enabled}) #orca_logo_notice() #end -#if(${bundled-bootstrap}) +#if(${bundled_bootstrap_enabled}) #bootstrap_notice() #end -#if(${bundled-dependencies}) +#if(${bundled_dependencies_enabled}) #** Note that this will fail the build if we don't have a license. update supplemental-models via setting '-Dlicense.debug.print.included' and looking in the generated LICENSE file @@ -221,7 +221,7 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace()) #set($mpl_1_1=[]) ## track jettys #set($jetty=false) -#set($jetty-with-crypt=false) +#set($jetty_with_crypt=false) ## track jruby #set($jruby=false) #foreach( ${dep} in ${projects} ) @@ -236,7 +236,7 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace()) #if(${dep.artifactId.startsWith("jetty")}) #set($jetty=true) #if(${dep.artifactId.equals("jetty")}) -#set($jetty-with-crypt=true) +#set($jetty_with_crypt=true) #end #end #if(${dep.artifactId.equals("log4j")}) @@ -309,7 +309,7 @@ For source see '${dep.url}'. #end ## Print out jetty #if(${jetty}) -#jetty_notice(${jetty-with-crypt}) +#jetty_notice(${jetty_with_crypt}) #end ## Now go through all the lists of Category-B licensed works and make sure we ## name them and give a URL for the project's home page. diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml b/hbase-resource-bundle/src/main/resources/supplemental-models.xml index e1378583a878..f1008c020300 100644 --- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml +++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml @@ -1943,397 +1943,6 @@ Copyright 2005 Sun Microsystems, Inc. and portions Copyright Apache Software Fou - - - org.jamon - jamon-runtime - Jamon runtime support classes - http://www.jamon.org/ - - - - http://www.mozilla.org/MPL/2.0 - Mozilla Public License Version 2.0 - repo - -Mozilla Public License Version 2.0 -================================== - -1. Definitions --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. - - - - - diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml index 7bd7b188cc32..8fdf48403ba3 100644 --- a/hbase-rest/pom.xml +++ b/hbase-rest/pom.xml @@ -200,8 +200,18 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params test diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java index dea6f2909875..d7f28d82d26a 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java @@ -53,10 +53,10 @@ import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; +import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser; import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException; -import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; import org.apache.hbase.thirdparty.org.eclipse.jetty.jmx.MBeanContainer; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; @@ -129,7 +129,7 @@ private static void printUsageAndExit(Options options, int exitCode) { System.exit(exitCode); } - void addCSRFFilter(ServletContextHandler ctxHandler, Configuration conf) { + private void addCSRFFilter(ServletContextHandler ctxHandler, Configuration conf) { restCSRFEnabled = conf.getBoolean(REST_CSRF_ENABLED_KEY, REST_CSRF_ENABLED_DEFAULT); if (restCSRFEnabled) { Map restCsrfParams = @@ -178,7 +178,7 @@ private static void parseCommandLine(String[] args, Configuration conf) { CommandLine commandLine = null; try { - commandLine = new PosixParser().parse(options, args); + commandLine = new DefaultParser().parse(options, args); } catch (ParseException e) { LOG.error("Could not parse: ", e); printUsageAndExit(options, -1); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/DummyFilter.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/DummyFilter.java index 4c9b20c6ace3..d6d50b8d7810 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/DummyFilter.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/DummyFilter.java @@ -48,7 +48,7 @@ public void doFilter(ServletRequest paramServletRequest, ServletResponse paramSe String path = request.getRequestURI(); LOG.info(path); - if (path.indexOf("/status/cluster") >= 0) { + if (path.contains("/status/cluster")) { LOG.info("Blocking cluster status request"); response.sendError(HttpServletResponse.SC_NOT_FOUND, "Cluster status cannot be requested."); } else { @@ -60,5 +60,4 @@ public void doFilter(ServletRequest paramServletRequest, ServletResponse paramSe @Override public void init(FilterConfig filterChain) throws ServletException { } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/MultiRowResourceTestBase.java similarity index 89% rename from hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java rename to hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/MultiRowResourceTestBase.java index 03c3fe068599..2eac0ad78aa6 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/MultiRowResourceTestBase.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -25,43 +25,29 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Base64.Encoder; -import java.util.Collection; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.rest.client.Client; import org.apache.hadoop.hbase.rest.client.Cluster; import org.apache.hadoop.hbase.rest.client.Response; import org.apache.hadoop.hbase.rest.model.CellModel; import org.apache.hadoop.hbase.rest.model.CellSetModel; import org.apache.hadoop.hbase.rest.model.RowModel; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.http.Header; import org.apache.http.message.BasicHeader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; -@Category({ RestTests.class, MediumTests.class }) -@RunWith(Parameterized.class) -public class TestMultiRowResource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMultiRowResource.class); +public class MultiRowResourceTestBase { private static final TableName TABLE = TableName.valueOf("TestRowResource"); private static final String CFA = "a"; @@ -82,19 +68,9 @@ public class TestMultiRowResource { private static Configuration conf; private static Header extraHdr = null; - private static boolean csrfEnabled = true; + protected static boolean csrfEnabled = true; - @Parameterized.Parameters - public static Collection data() { - return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; - } - - public TestMultiRowResource(Boolean csrf) { - csrfEnabled = csrf; - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { + protected static void initialize() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setBoolean(RESTServer.REST_CSRF_ENABLED_KEY, csrfEnabled); if (csrfEnabled) { @@ -108,14 +84,17 @@ public static void setUpBeforeClass() throws Exception { if (admin.tableExists(TABLE)) { return; } - HTableDescriptor htd = new HTableDescriptor(TABLE); - htd.addFamily(new HColumnDescriptor(CFA)); - htd.addFamily(new HColumnDescriptor(CFB)); - admin.createTable(htd); + TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(TABLE); + ColumnFamilyDescriptor columnFamilyDescriptor = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFA)).build(); + tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); + columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFB)).build(); + tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); + admin.createTable(tableDescriptorBuilder.build()); } - @AfterClass - public static void tearDownAfterClass() throws Exception { + @AfterAll + public static void tearDownAfterAll() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); } @@ -191,9 +170,9 @@ public void testMultiCellGetJSONB64() throws IOException { path.append("/"); path.append(TABLE); path.append("/multiget/?row="); - path.append(encoder.encodeToString(ROW_1.getBytes("UTF-8"))); + path.append(encoder.encodeToString(ROW_1.getBytes(StandardCharsets.UTF_8))); path.append("&row="); - path.append(encoder.encodeToString(ROW_2.getBytes("UTF-8"))); + path.append(encoder.encodeToString(ROW_2.getBytes(StandardCharsets.UTF_8))); path.append("&e=b64"); // Specify encoding via query string Response response = client.get(path.toString(), Constants.MIMETYPE_JSON); @@ -204,9 +183,9 @@ public void testMultiCellGetJSONB64() throws IOException { path.append("/"); path.append(TABLE); path.append("/multiget/?row="); - path.append(encoder.encodeToString(ROW_1.getBytes("UTF-8"))); + path.append(encoder.encodeToString(ROW_1.getBytes(StandardCharsets.UTF_8))); path.append("&row="); - path.append(encoder.encodeToString(ROW_2.getBytes("UTF-8"))); + path.append(encoder.encodeToString(ROW_2.getBytes(StandardCharsets.UTF_8))); Header[] headers = new Header[] { new BasicHeader("Accept", Constants.MIMETYPE_JSON), new BasicHeader("Encoding", "b64") // Specify encoding via header @@ -371,7 +350,7 @@ public void testMultiCellGetFilterJSON() throws IOException { // If the filter is used, then we get the same result String positivePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder - .encodeToString("PrefixFilter('testrow')".getBytes(StandardCharsets.UTF_8.toString()))); + .encodeToString("PrefixFilter('testrow')".getBytes(StandardCharsets.UTF_8))); response = client.get(positivePath, Constants.MIMETYPE_JSON); checkMultiCellGetJSON(response); @@ -383,7 +362,7 @@ public void testMultiCellGetFilterJSON() throws IOException { // This filter doesn't match the found rows String negativePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder - .encodeToString("PrefixFilter('notfound')".getBytes(StandardCharsets.UTF_8.toString()))); + .encodeToString("PrefixFilter('notfound')".getBytes(StandardCharsets.UTF_8))); response = client.get(negativePath, Constants.MIMETYPE_JSON); assertEquals(404, response.getCode()); @@ -409,5 +388,4 @@ public void testMultiCellGetFilterJSON() throws IOException { client.delete(row_5_url, extraHdr); client.delete(row_6_url, extraHdr); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/RowResourceBase.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/RowResourceBase.java index 4d5fe8ea64ac..239d91dd80f4 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/RowResourceBase.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/RowResourceBase.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; @@ -45,10 +45,10 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.http.Header; import org.apache.http.message.BasicHeader; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; @@ -83,7 +83,7 @@ public class RowResourceBase { protected static Configuration conf; protected static ObjectMapper jsonMapper; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); TEST_UTIL.startMiniCluster(3); @@ -96,13 +96,13 @@ public static void setUpBeforeClass() throws Exception { client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); } - @Before + @BeforeEach public void beforeMethod() throws Exception { Admin admin = TEST_UTIL.getAdmin(); if (admin.tableExists(TABLE_NAME)) { @@ -114,7 +114,7 @@ public void beforeMethod() throws Exception { admin.createTable(htd); } - @After + @AfterEach public void afterMethod() throws Exception { Admin admin = TEST_UTIL.getAdmin(); if (admin.tableExists(TABLE_NAME)) { diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/SchemaResourceTestBase.java similarity index 85% rename from hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java rename to hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/SchemaResourceTestBase.java index 191eae70d3fb..0f5a4a452e51 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/SchemaResourceTestBase.java @@ -17,19 +17,16 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.StringWriter; -import java.util.Collection; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -44,21 +41,14 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.http.Header; import org.apache.http.message.BasicHeader; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@Category({ RestTests.class, MediumTests.class }) -@RunWith(Parameterized.class) -public class TestSchemaResource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSchemaResource.class); +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) +public class SchemaResourceTestBase { private static String TABLE1 = "TestSchemaResource1"; private static String TABLE2 = "TestSchemaResource2"; @@ -71,19 +61,9 @@ public class TestSchemaResource { private static TestTableSchemaModel testTableSchemaModel; private static Header extraHdr = null; - private static boolean csrfEnabled = true; - - @Parameterized.Parameters - public static Collection parameters() { - return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; - } - - public TestSchemaResource(Boolean csrf) { - csrfEnabled = csrf; - } + protected static boolean csrfEnabled = true; - @BeforeClass - public static void setUpBeforeClass() throws Exception { + protected static void initialize() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setBoolean(RESTServer.REST_CSRF_ENABLED_KEY, csrfEnabled); if (csrfEnabled) { @@ -97,13 +77,13 @@ public static void setUpBeforeClass() throws Exception { context = JAXBContext.newInstance(ColumnSchemaModel.class, TableSchemaModel.class); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); } - @After + @AfterEach public void tearDown() throws Exception { Admin admin = TEST_UTIL.getAdmin(); @@ -136,8 +116,8 @@ public void testTableCreateAndDeleteXML() throws IOException, JAXBException { Response response; Admin admin = TEST_UTIL.getAdmin(); - assertFalse("Table " + TABLE1 + " should not exist", - admin.tableExists(TableName.valueOf(TABLE1))); + assertFalse(admin.tableExists(TableName.valueOf(TABLE1)), + "Table " + TABLE1 + " should not exist"); // create the table model = testTableSchemaModel.buildTestModel(TABLE1); @@ -149,8 +129,8 @@ public void testTableCreateAndDeleteXML() throws IOException, JAXBException { } response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model), extraHdr); - assertEquals("put failed with csrf " + (csrfEnabled ? "enabled" : "disabled"), 201, - response.getCode()); + assertEquals(201, response.getCode(), + "put failed with csrf " + (csrfEnabled ? "enabled" : "disabled")); // recall the same put operation but in read-only mode conf.set("hbase.rest.readonly", "true"); @@ -210,8 +190,8 @@ public void testTableCreateAndDeletePB() throws IOException { } response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput(), extraHdr); - assertEquals("put failed with csrf " + (csrfEnabled ? "enabled" : "disabled"), 201, - response.getCode()); + assertEquals(201, response.getCode(), + "put failed with csrf " + (csrfEnabled ? "enabled" : "disabled")); // recall the same put operation but in read-only mode conf.set("hbase.rest.readonly", "true"); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestDeleteRow.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestDeleteRow.java index 9893a9ef67d2..e19564a4d1cc 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestDeleteRow.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestDeleteRow.java @@ -17,25 +17,20 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import javax.xml.bind.JAXBException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.rest.client.Response; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestDeleteRow extends RowResourceBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestDeleteRow.class); - @Test public void testDeleteNonExistentColumn() throws Exception { Response response = putValueJson(TABLE, ROW_1, COLUMN_1, VALUE_1); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java index fbff87c3d0c1..e98230d99618 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGZIPResponseWrapper.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -27,22 +27,17 @@ import java.io.IOException; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.rest.filter.GZIPResponseStream; import org.apache.hadoop.hbase.rest.filter.GZIPResponseWrapper; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestGZIPResponseWrapper { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestGZIPResponseWrapper.class); - private final HttpServletResponse response = mock(HttpServletResponse.class); private final GZIPResponseWrapper wrapper = new GZIPResponseWrapper(response); @@ -117,5 +112,4 @@ public void testSendError() throws IOException { wrapper.sendError(404, "error message"); verify(response).sendError(404, "error message"); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java index b20baea9df8c..c17b6bf0af2f 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -29,7 +29,6 @@ import java.util.List; import javax.xml.bind.JAXBException; import org.apache.hadoop.hbase.CompatibilityFactory; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.rest.client.Response; import org.apache.hadoop.hbase.rest.model.CellModel; @@ -37,20 +36,17 @@ import org.apache.hadoop.hbase.rest.model.RowModel; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.test.MetricsAssertHelper; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.http.Header; import org.apache.http.message.BasicHeader; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(LargeTests.TAG) public class TestGetAndPutResource extends RowResourceBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestGetAndPutResource.class); private static final MetricsAssertHelper METRICS_ASSERT = CompatibilityFactory.getInstance(MetricsAssertHelper.class); @@ -310,8 +306,8 @@ public void testLatestCellGetJSON() throws IOException { assertEquals(200, response.getCode()); assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); CellSetModel cellSet = jsonMapper.readValue(response.getBody(), CellSetModel.class); - assertTrue(cellSet.getRows().size() == 1); - assertTrue(cellSet.getRows().get(0).getCells().size() == 1); + assertEquals(1, cellSet.getRows().size()); + assertEquals(1, cellSet.getRows().get(0).getCells().size()); CellModel cell = cellSet.getRows().get(0).getCells().get(0); assertEquals(VALUE_2, Bytes.toString(cell.getValue())); assertEquals(2L, cell.getTimestamp()); @@ -623,8 +619,8 @@ public void testMultiColumnGetXML() throws Exception { assertEquals(200, response.getCode()); CellSetModel cellSet = (CellSetModel) xmlUnmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); - assertTrue(cellSet.getRows().size() == 1); - assertTrue(cellSet.getRows().get(0).getCells().size() == 3); + assertEquals(1, cellSet.getRows().size()); + assertEquals(3, cellSet.getRows().get(0).getCells().size()); List cells = cellSet.getRows().get(0).getCells(); assertTrue(containsCellModel(cells, COLUMN_1, VALUE_1)); @@ -681,7 +677,7 @@ public void testSuffixGlobbingXMLWithNewScanner() throws IOException, JAXBExcept assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); CellSetModel cellSet = (CellSetModel) xmlUnmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); - assertTrue(cellSet.getRows().size() == 2); + assertEquals(2, cellSet.getRows().size()); response = deleteRow(TABLE, ROW_1); assertEquals(200, response.getCode()); @@ -725,9 +721,9 @@ public void testSuffixGlobbingXML() throws IOException, JAXBException { CellSetModel cellSet = (CellSetModel) xmlUnmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); List rows = cellSet.getRows(); - assertTrue(rows.size() == 2); + assertEquals(2, rows.size()); for (RowModel row : rows) { - assertTrue(row.getCells().size() == 1); + assertEquals(1, row.getCells().size()); assertEquals(COLUMN_1, Bytes.toString(row.getCells().get(0).getColumn())); } response = deleteRow(TABLE, ROW_1); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java index fa3c0a228b8c..7ffd16799085 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGzipFilter.java @@ -17,15 +17,14 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -42,19 +41,15 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.http.Header; import org.apache.http.message.BasicHeader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestGzipFilter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestGzipFilter.class); - private static final TableName TABLE = TableName.valueOf("TestGzipFilter"); private static final String CFA = "a"; private static final String COLUMN_1 = CFA + ":1"; @@ -66,7 +61,7 @@ public class TestGzipFilter { private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); private static Client client; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(); REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); @@ -80,7 +75,7 @@ public static void setUpBeforeClass() throws Exception { admin.createTable(htd); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); @@ -143,5 +138,4 @@ void testScannerResultCodes() throws Exception { response = client.get(scannerUrl); assertEquals(204, response.getCode()); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResourceCsrfDisabled.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResourceCsrfDisabled.java new file mode 100644 index 000000000000..c3b788f7d354 --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResourceCsrfDisabled.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest; + +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RestTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) +public class TestMultiRowResourceCsrfDisabled extends MultiRowResourceTestBase { + + @BeforeAll + public static void setUpBeforeAll() throws Exception { + csrfEnabled = false; + initialize(); + } +} diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResourceCsrfEnabled.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResourceCsrfEnabled.java new file mode 100644 index 000000000000..e56286a2ca66 --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResourceCsrfEnabled.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest; + +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RestTests; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) +public class TestMultiRowResourceCsrfEnabled extends MultiRowResourceTestBase { + + @BeforeAll + public static void setUpBeforeAll() throws Exception { + csrfEnabled = true; + initialize(); + } +} diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java index 2d09ae0da4a2..480dab4a347d 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; @@ -34,7 +34,6 @@ import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -52,30 +51,26 @@ import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.http.Header; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestNamespacesInstanceResource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestNamespacesInstanceResource.class); - - private static String NAMESPACE1 = "TestNamespacesInstanceResource1"; - private static Map NAMESPACE1_PROPS = new HashMap<>(); - private static String NAMESPACE2 = "TestNamespacesInstanceResource2"; - private static Map NAMESPACE2_PROPS = new HashMap<>(); - private static String NAMESPACE3 = "TestNamespacesInstanceResource3"; - private static Map NAMESPACE3_PROPS = new HashMap<>(); - private static String NAMESPACE4 = "TestNamespacesInstanceResource4"; - private static Map NAMESPACE4_PROPS = new HashMap<>(); + + private static final String NAMESPACE1 = "TestNamespacesInstanceResource1"; + private static final Map NAMESPACE1_PROPS = new HashMap<>(); + private static final String NAMESPACE2 = "TestNamespacesInstanceResource2"; + private static final Map NAMESPACE2_PROPS = new HashMap<>(); + private static final String NAMESPACE3 = "TestNamespacesInstanceResource3"; + private static final Map NAMESPACE3_PROPS = new HashMap<>(); + private static final String NAMESPACE4 = "TestNamespacesInstanceResource4"; + private static final Map NAMESPACE4_PROPS = new HashMap<>(); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); @@ -85,7 +80,7 @@ public class TestNamespacesInstanceResource { private static TestNamespacesInstanceModel testNamespacesInstanceModel; protected static ObjectMapper jsonMapper; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); TEST_UTIL.startMiniCluster(); @@ -103,7 +98,7 @@ public static void setUpBeforeClass() throws Exception { NAMESPACE4_PROPS.put("key4b", "value4b"); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); @@ -116,8 +111,9 @@ private static byte[] toXML(NamespacesInstanceModel model) throws JAXBException } @SuppressWarnings("unchecked") - private static T fromXML(byte[] content) throws JAXBException { - return (T) context.createUnmarshaller().unmarshal(new ByteArrayInputStream(content)); + private static T fromXML(byte[] content, Class clazz) throws JAXBException { + JAXBContext jaxbContext = JAXBContext.newInstance(clazz); + return (T) jaxbContext.createUnmarshaller().unmarshal(new ByteArrayInputStream(content)); } private NamespaceDescriptor findNamespace(Admin admin, String namespaceName) throws IOException { @@ -136,7 +132,7 @@ private void checkNamespaceProperties(NamespaceDescriptor nd, Map namespaceProps, Map testProps) { - assertTrue(namespaceProps.size() == testProps.size()); + assertEquals(namespaceProps.size(), testProps.size()); for (String key : testProps.keySet()) { assertEquals(testProps.get(key), namespaceProps.get(key)); } @@ -205,7 +201,7 @@ public void testGetNamespaceTablesAndCannotDeleteNamespace() throws IOException, response = client.get(namespacePath, Constants.MIMETYPE_XML); assertEquals(200, response.getCode()); - NamespacesInstanceModel model = fromXML(response.getBody()); + NamespacesInstanceModel model = fromXML(response.getBody(), NamespacesInstanceModel.class); checkNamespaceProperties(model.getProperties(), nsProperties); response = client.get(namespacePath, Constants.MIMETYPE_JSON); @@ -225,7 +221,7 @@ public void testGetNamespaceTablesAndCannotDeleteNamespace() throws IOException, response = client.get(namespacePath, Constants.MIMETYPE_XML); assertEquals(200, response.getCode()); - TableListModel tablemodel = fromXML(response.getBody()); + TableListModel tablemodel = fromXML(response.getBody(), TableListModel.class); checkNamespaceTables(tablemodel.getTables(), nsTables); response = client.get(namespacePath, Constants.MIMETYPE_JSON); @@ -245,7 +241,6 @@ public void testGetNamespaceTablesAndCannotDeleteNamespace() throws IOException, assertEquals(503, response.getCode()); } - @Ignore("HBASE-19210") @Test public void testInvalidNamespacePostsAndPuts() throws IOException, JAXBException { String namespacePath1 = "/namespaces/" + NAMESPACE1; diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesResource.java index 4802d230fac5..a0489e7ee7a1 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesResource.java @@ -17,15 +17,14 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.io.ByteArrayInputStream; import java.io.IOException; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.client.Admin; @@ -37,17 +36,14 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestNamespacesResource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestNamespacesResource.class); private static String NAMESPACE1 = "TestNamespacesInstanceResource1"; private static String NAMESPACE2 = "TestNamespacesInstanceResource2"; @@ -59,7 +55,7 @@ public class TestNamespacesResource { private static Configuration conf; private static TestNamespacesModel testNamespacesModel; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); TEST_UTIL.startMiniCluster(); @@ -69,7 +65,7 @@ public static void setUpBeforeClass() throws Exception { context = JAXBContext.newInstance(NamespacesModel.class); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestRESTServerSSL.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestRESTServerSSL.java index 5731dd94fc66..a6f98a0860b2 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestRESTServerSSL.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestRESTServerSSL.java @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.File; import java.lang.reflect.Method; @@ -25,7 +26,6 @@ import java.security.cert.X509Certificate; import java.util.Optional; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; import org.apache.hadoop.hbase.rest.client.Client; @@ -33,23 +33,20 @@ import org.apache.hadoop.hbase.rest.client.Response; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.apache.http.client.ClientProtocolException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestRESTServerSSL { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRESTServerSSL.class); - private static final Logger LOG = LoggerFactory.getLogger(TestRESTServerSSL.class); private static final String KEY_STORE_PASSWORD = "myKSPassword"; @@ -75,7 +72,7 @@ private static void initializeAlgorithmId() { } } - @BeforeClass + @BeforeAll public static void beforeClass() throws Exception { initializeAlgorithmId(); keyDir = initKeystoreDir(); @@ -94,14 +91,14 @@ public static void beforeClass() throws Exception { TEST_UTIL.startMiniCluster(); } - @AfterClass + @AfterAll public static void afterClass() throws Exception { // this will also delete the generated test keystore / teststore files, // as we were placing them under the dataTestDir used by the minicluster TEST_UTIL.shutdownMiniCluster(); } - @Before + @BeforeEach public void beforeEachTest() { conf = new Configuration(TEST_UTIL.getConfiguration()); conf.set(Constants.REST_SSL_ENABLED, "true"); @@ -110,7 +107,7 @@ public void beforeEachTest() { conf.set(Constants.REST_SSL_TRUSTSTORE_PASSWORD, TRUST_STORE_PASSWORD); } - @After + @AfterEach public void tearDownAfterTest() { REST_TEST_UTIL.shutdownServletContainer(); } @@ -129,14 +126,14 @@ public void testSslConnection() throws Exception { response.getHeader("Content-Security-Policy")); } - @Test(expected = org.apache.http.client.ClientProtocolException.class) + @Test public void testNonSslClientDenied() throws Exception { startRESTServerWithDefaultKeystoreType(); Cluster localCluster = new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()); Client nonSslClient = new Client(localCluster, false); - nonSslClient.get("/version"); + assertThrows(ClientProtocolException.class, () -> nonSslClient.get("/version")); } @Test @@ -214,5 +211,4 @@ private void startRESTServer(String storeType) throws Exception { sslClient = new Client(localCluster, getTruststoreFilePath(storeType), Optional.of(TRUST_STORE_PASSWORD), Optional.of(storeType)); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestResourceFilter.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestResourceFilter.java index 4168c75a7c56..1bbe9954efc8 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestResourceFilter.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestResourceFilter.java @@ -17,33 +17,28 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.rest.client.Client; import org.apache.hadoop.hbase.rest.client.Cluster; import org.apache.hadoop.hbase.rest.client.Response; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestResourceFilter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestResourceFilter.class); - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); private static Client client; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().set(Constants.FILTER_CLASSES, DummyFilter.class.getName()); TEST_UTIL.startMiniCluster(); @@ -51,7 +46,7 @@ public static void setUpBeforeClass() throws Exception { client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java index 646328b1f49e..7f7c527cc062 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -36,7 +36,6 @@ import javax.xml.bind.Unmarshaller; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -58,22 +57,15 @@ import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.http.Header; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Category({ RestTests.class, MediumTests.class }) -public class TestScannerResource { +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestScannerResource.class); +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) +public class TestScannerResource { - private static final Logger LOG = LoggerFactory.getLogger(TestScannerResource.class); private static final TableName TABLE = TableName.valueOf("TestScannerResource"); private static final TableName TABLE_TO_BE_DISABLED = TableName.valueOf("ScannerResourceDisable"); private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist"; @@ -168,7 +160,7 @@ private static int fullTableScan(ScannerModel model) throws IOException { return count; } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); TEST_UTIL.startMiniCluster(); @@ -195,7 +187,7 @@ public static void setUpBeforeClass() throws Exception { admin.createTable(htd); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); @@ -384,7 +376,7 @@ public void testTableScanWithTableDisable() throws IOException { assertNotNull(scannerURI); TEST_UTIL.getAdmin().disableTable(TABLE_TO_BE_DISABLED); response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF); - assertTrue("got " + response.getCode(), response.getCode() == 410); + assertEquals(410, response.getCode(), "got " + response.getCode()); } @Test diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java index 88e685ec0778..ee27be9523a0 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.StringWriter; @@ -32,7 +32,6 @@ import javax.xml.bind.Unmarshaller; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.CompareOperator; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -72,19 +71,16 @@ import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestScannersWithFilters { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestScannersWithFilters.class); private static final Logger LOG = LoggerFactory.getLogger(TestScannersWithFilters.class); @@ -119,7 +115,7 @@ public class TestScannersWithFilters { private static long numRows = (long) ROWS_ONE.length + ROWS_TWO.length; private static long colsPerRow = (long) FAMILIES.length * QUALIFIERS_ONE.length; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(3); REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); @@ -204,7 +200,7 @@ public static void setUpBeforeClass() throws Exception { } } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); @@ -230,13 +226,12 @@ private static void verifyScan(Scan s, long expectedRows, long expectedKeys) thr (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); int rows = cells.getRows().size(); - assertEquals( - "Unexpected number of rows! Expected " + expectedRows + " total but scanned " + rows, - expectedRows, rows); + assertEquals(expectedRows, rows, + "Unexpected number of rows! Expected " + expectedRows + " total but scanned " + rows); for (RowModel row : cells.getRows()) { int count = row.getCells().size(); - assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + count, - expectedKeys, count); + assertEquals(expectedKeys, count, + "Expected " + expectedKeys + " keys per row but " + "returned " + count); } // delete the scanner @@ -282,18 +277,18 @@ private static void verifyScanFull(Scan s, KeyValue[] kvs) throws Exception { break; } - assertTrue("Scanned too many keys! Only expected " + kvs.length - + " total but already scanned " + (cells.size() + idx), kvs.length >= idx + cells.size()); + assertTrue(kvs.length >= idx + cells.size(), "Scanned too many keys! Only expected " + + kvs.length + " total but already scanned " + (cells.size() + idx)); for (CellModel cell : cells) { - assertTrue("Row mismatch", Bytes.equals(rowModel.getKey(), CellUtil.cloneRow(kvs[idx]))); + assertTrue(Bytes.equals(rowModel.getKey(), CellUtil.cloneRow(kvs[idx])), "Row mismatch"); byte[][] split = CellUtil.parseColumn(cell.getColumn()); - assertTrue("Family mismatch", Bytes.equals(split[0], CellUtil.cloneFamily(kvs[idx]))); - assertTrue("Qualifier mismatch", Bytes.equals(split[1], CellUtil.cloneQualifier(kvs[idx]))); - assertTrue("Value mismatch", Bytes.equals(cell.getValue(), CellUtil.cloneValue(kvs[idx]))); + assertTrue(Bytes.equals(split[0], CellUtil.cloneFamily(kvs[idx])), "Family mismatch"); + assertTrue(Bytes.equals(split[1], CellUtil.cloneQualifier(kvs[idx])), "Qualifier mismatch"); + assertTrue(Bytes.equals(cell.getValue(), CellUtil.cloneValue(kvs[idx])), "Value mismatch"); idx++; } } - assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, kvs.length, idx); + assertEquals(kvs.length, idx, "Expected " + kvs.length + " total keys but scanned " + idx); } private static void verifyScanNoEarlyOut(Scan s, long expectedRows, long expectedKeys) @@ -334,12 +329,12 @@ private static void verifyScanNoEarlyOut(Scan s, long expectedRows, long expecte break; } - assertTrue("Scanned too many rows! Only expected " + expectedRows - + " total but already scanned " + (j + 1), expectedRows > j); - assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + cells.size(), - expectedKeys, cells.size()); + assertTrue(expectedRows > j, "Scanned too many rows! Only expected " + expectedRows + + " total but already scanned " + (j + 1)); + assertEquals(expectedKeys, cells.size(), + "Expected " + expectedKeys + " keys per row but " + "returned " + cells.size()); } - assertEquals("Expected " + expectedRows + " rows but scanned " + j + " rows", expectedRows, j); + assertEquals(expectedRows, j, "Expected " + expectedRows + " rows but scanned " + j + " rows"); } @Test diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java index a17ebb1789d9..7f4dd3f7a490 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -33,7 +33,6 @@ import javax.xml.bind.Unmarshaller; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -63,17 +62,14 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestScannersWithLabels { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestScannersWithLabels.class); private static final TableName TABLE = TableName.valueOf("TestScannersWithLabels"); private static final String CFA = "a"; @@ -129,7 +125,7 @@ private static int countCellSet(CellSetModel model) { return count; } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); conf = TEST_UTIL.getConfiguration(); @@ -160,7 +156,7 @@ public static void setUpBeforeClass() throws Exception { insertData(TABLE, COLUMN_2, 0.5); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResourceCsrfDisabled.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResourceCsrfDisabled.java new file mode 100644 index 000000000000..13faea51858c --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResourceCsrfDisabled.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest; + +import org.junit.jupiter.api.BeforeAll; + +public class TestSchemaResourceCsrfDisabled extends SchemaResourceTestBase { + + @BeforeAll + public static void setUpBeforeAll() throws Exception { + csrfEnabled = false; + initialize(); + } +} diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResourceCsrfEnabled.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResourceCsrfEnabled.java new file mode 100644 index 000000000000..0b1a9e5ea719 --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResourceCsrfEnabled.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest; + +import org.junit.jupiter.api.BeforeAll; + +public class TestSchemaResourceCsrfEnabled extends SchemaResourceTestBase { + + @BeforeAll + public static void setUpBeforeAll() throws Exception { + csrfEnabled = true; + initialize(); + } +} diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java index 48ef22355e93..38d9637794f8 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecureRESTServer.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hbase.rest; import static org.apache.hadoop.hbase.rest.RESTServlet.HBASE_REST_SUPPORT_PROXYUSER; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; @@ -30,7 +30,6 @@ import java.security.PrivilegedExceptionAction; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.StartMiniClusterOption; @@ -86,11 +85,10 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,13 +99,10 @@ * Test class for SPNEGO authentication on the HttpServer. Uses Kerby's MiniKDC and Apache * HttpComponents to verify that a simple Servlet is reachable via SPNEGO and unreachable w/o. */ -@Category({ MiscTests.class, MediumTests.class }) +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) public class TestSecureRESTServer { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSecureRESTServer.class); - private static final Logger LOG = LoggerFactory.getLogger(TestSecureRESTServer.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final HBaseRESTTestingUtility REST_TEST = new HBaseRESTTestingUtility(); @@ -130,7 +125,7 @@ public class TestSecureRESTServer { private static File wheelKeytab; private static File serviceKeytab; - @BeforeClass + @BeforeAll public static void setupServer() throws Exception { final File target = new File(System.getProperty("user.dir"), "target"); assertTrue(target.exists()); @@ -235,7 +230,7 @@ public Void run() throws Exception { instertData(); } - @AfterClass + @AfterAll public static void stopServer() throws Exception { try { if (null != server) { @@ -356,7 +351,7 @@ public void testProxy(String extraArgs, String PRINCIPAL, File keytab, int respo public String run() throws Exception { try (CloseableHttpResponse response = client.execute(get, context)) { final int statusCode = response.getStatusLine().getStatusCode(); - assertEquals(response.getStatusLine().toString(), responseCode, statusCode); + assertEquals(responseCode, statusCode, response.getStatusLine().toString()); HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity); } @@ -414,8 +409,8 @@ public Void run() throws Exception { try (CloseableHttpResponse response = client.execute(put, context)) { final int statusCode = response.getStatusLine().getStatusCode(); HttpEntity entity = response.getEntity(); - assertEquals("Got response: " + EntityUtils.toString(entity), - HttpURLConnection.HTTP_FORBIDDEN, statusCode); + assertEquals(HttpURLConnection.HTTP_FORBIDDEN, statusCode, + "Got response: " + EntityUtils.toString(entity)); } return null; } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecurityHeadersFilter.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecurityHeadersFilter.java index 6a24d6ac59bd..549a645deb11 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecurityHeadersFilter.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSecurityHeadersFilter.java @@ -22,30 +22,25 @@ import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsEqual.equalTo; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.rest.client.Client; import org.apache.hadoop.hbase.rest.client.Cluster; import org.apache.hadoop.hbase.rest.client.Response; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; -import org.junit.After; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestSecurityHeadersFilter { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSecurityHeadersFilter.class); - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); private static Client client; - @After + @AfterEach public void tearDown() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java index 28bf14d7a751..77c7b079831f 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java @@ -17,17 +17,16 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.ByteArrayInputStream; import java.io.IOException; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; @@ -38,23 +37,15 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestStatusResource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStatusResource.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestStatusResource.class); - private static final byte[] META_REGION_NAME = Bytes.toBytes(TableName.META_TABLE_NAME + ",,1"); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -85,7 +76,7 @@ private static void validate(StorageClusterStatusModel model) { assertTrue(foundMeta); } - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); TEST_UTIL.startMiniCluster(); @@ -104,7 +95,7 @@ public boolean evaluate() throws IOException { }); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java index 097d3566435c..c58e39f5b290 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -29,7 +29,6 @@ import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; @@ -50,21 +49,17 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestTableResource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableResource.class); - private static final Logger LOG = LoggerFactory.getLogger(TestTableResource.class); private static final TableName TABLE = TableName.valueOf("TestTableResource"); @@ -78,7 +73,7 @@ public class TestTableResource { private static Client client; private static JAXBContext context; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(3); REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); @@ -115,11 +110,11 @@ public static void setUpBeforeClass() throws Exception { // should have four regions now assertEquals(NUM_REGIONS, m.size()); regionMap = m; - LOG.error("regions: " + regionMap); + LOG.error("regions: {}", regionMap); regionLocator.close(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); @@ -146,14 +141,14 @@ void checkTableInfo(TableInfoModel model) { while (regions.hasNext()) { TableRegionModel region = regions.next(); boolean found = false; - LOG.debug("looking for region " + region.getName()); + LOG.debug("looking for region {}", region.getName()); for (HRegionLocation e : regionMap) { HRegionInfo hri = e.getRegionInfo(); // getRegionNameAsString uses Bytes.toStringBinary which escapes some non-printable // characters String hriRegionName = Bytes.toString(hri.getRegionName()); String regionName = region.getName(); - LOG.debug("comparing to region " + hriRegionName); + LOG.debug("comparing to region {}", hriRegionName); if (hriRegionName.equals(regionName)) { found = true; byte[] startKey = hri.getStartKey(); @@ -169,7 +164,7 @@ void checkTableInfo(TableInfoModel model) { break; } } - assertTrue("Couldn't find region " + region.getName(), found); + assertTrue(found, "Couldn't find region " + region.getName()); } } @@ -261,5 +256,4 @@ public void testTableNotFound() throws IOException { Response response2 = client.get("/" + notExistTable + "/regions", Constants.MIMETYPE_XML); assertEquals(404, response2.getCode()); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java index 56afb68537b9..b43dd4e883bc 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; @@ -46,7 +46,6 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.parsers.SAXParserFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -64,22 +63,19 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.xml.sax.InputSource; import org.xml.sax.XMLReader; import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestTableScan { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableScan.class); private static final TableName TABLE = TableName.valueOf("TestScanResource"); private static final String CFA = "a"; @@ -97,7 +93,7 @@ public class TestTableScan { private static final Encoder base64UrlEncoder = java.util.Base64.getUrlEncoder().withoutPadding(); private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.set(Constants.CUSTOM_FILTERS, "CustomFilter:" + CustomFilter.class.getName()); @@ -116,7 +112,7 @@ public static void setUpBeforeClass() throws Exception { } } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { TEST_UTIL.getAdmin().disableTable(TABLE); TEST_UTIL.getAdmin().deleteTable(TABLE); @@ -271,8 +267,8 @@ public void testScanUsingListenerUnmarshallerXML() throws Exception { final ClientSideCellSetModel.Listener listener = new ClientSideCellSetModel.Listener() { @Override public void handleRowModel(ClientSideCellSetModel helper, RowModel row) { - assertTrue(row.getKey() != null); - assertTrue(row.getCells().size() > 0); + assertNotNull(row.getKey()); + assertFalse(row.getCells().isEmpty()); } }; @@ -377,8 +373,8 @@ public void testSimpleScannerProtobuf() throws Exception { private void checkRowsNotNull(CellSetModel model) { for (RowModel row : model.getRows()) { - assertTrue(row.getKey() != null); - assertTrue(row.getCells().size() > 0); + assertNotNull(row.getKey()); + assertFalse(row.getCells().isEmpty()); } } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java index abca65a56756..d5e4b8a02473 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestVersionResource.java @@ -17,16 +17,16 @@ */ package org.apache.hadoop.hbase.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; import java.io.IOException; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.rest.client.Client; import org.apache.hadoop.hbase.rest.client.Cluster; @@ -37,24 +37,20 @@ import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.VersionInfo; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; -@Category({ RestTests.class, MediumTests.class }) +@Tag(RestTests.TAG) +@Tag(MediumTests.TAG) public class TestVersionResource { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestVersionResource.class); - private static final Logger LOG = LoggerFactory.getLogger(TestVersionResource.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -62,7 +58,7 @@ public class TestVersionResource { private static Client client; private static JAXBContext context; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(); REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); @@ -70,7 +66,7 @@ public static void setUpBeforeClass() throws Exception { context = JAXBContext.newInstance(VersionModel.class, StorageClusterVersionModel.class); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); @@ -111,7 +107,7 @@ public void testGetStargateVersionText() throws IOException { assertEquals(200, response.getCode()); assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type")); String body = Bytes.toString(response.getBody()); - assertTrue(body.length() > 0); + assertFalse(body.isEmpty()); assertTrue(body.contains(RESTServlet.VERSION_STRING)); assertTrue(body.contains(System.getProperty("java.vm.vendor"))); assertTrue(body.contains(System.getProperty("java.version"))); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdminRetries.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdminRetries.java index 81626d57e273..1f806190513c 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdminRetries.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteAdminRetries.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.rest.client; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; @@ -37,21 +36,17 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Tests {@link RemoteAdmin} retries. */ -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestRemoteAdminRetries { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRemoteAdminRetries.class); - private static final int SLEEP_TIME = 50; private static final int RETRIES = 3; private static final long MAX_TIME = SLEEP_TIME * (RETRIES - 1); @@ -61,7 +56,7 @@ public class TestRemoteAdminRetries { private RemoteAdmin remoteAdmin; private Client client; - @Before + @BeforeEach public void setup() throws Exception { client = mock(Client.class); Response response = new Response(509); @@ -166,8 +161,7 @@ private void testTimedOutCall(CallExecutor callExecutor) throws Exception { assertTrue((EnvironmentEdgeManager.currentTime() - start) > MAX_TIME); } - private static interface CallExecutor { + private interface CallExecutor { void run() throws Exception; } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteHTableRetries.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteHTableRetries.java index abdff1acdcdd..f00387c59f06 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteHTableRetries.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteHTableRetries.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.rest.client; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -30,7 +30,6 @@ import java.util.Arrays; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -40,20 +39,17 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; /** * Test RemoteHTable retries. */ -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestRemoteHTableRetries { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRemoteHTableRetries.class); private static final int SLEEP_TIME = 50; private static final int RETRIES = 3; @@ -69,7 +65,7 @@ public class TestRemoteHTableRetries { private Client client; private RemoteHTable remoteTable; - @Before + @BeforeEach public void setup() throws Exception { client = mock(Client.class); Response response = new Response(509); @@ -85,7 +81,7 @@ public void setup() throws Exception { remoteTable = new RemoteHTable(client, TEST_UTIL.getConfiguration(), "MyTable"); } - @After + @AfterEach public void tearDownAfterClass() throws Exception { remoteTable.close(); } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java index a0ec0bf1640b..7686cfea1957 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.hbase.rest.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -30,7 +30,6 @@ import java.util.List; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -45,25 +44,22 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.rest.HBaseRESTTestingUtility; import org.apache.hadoop.hbase.rest.RESTServlet; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.http.Header; import org.apache.http.message.BasicHeader; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({ RestTests.class, MediumTests.class }) +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(RestTests.TAG) +@Tag(LargeTests.TAG) public class TestRemoteTable { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRemoteTable.class); // Verify that invalid URL characters and arbitrary bytes are escaped when // constructing REST URLs per HBASE-7621. RemoteHTable should support row keys @@ -104,13 +100,13 @@ public class TestRemoteTable { private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); private RemoteHTable remoteTable; - @BeforeClass + @BeforeAll public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(); REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); } - @Before + @BeforeEach public void before() throws Exception { Admin admin = TEST_UTIL.getAdmin(); if (admin.tableExists(TABLE)) { @@ -140,12 +136,12 @@ public void before() throws Exception { TEST_UTIL.getConfiguration(), TABLE.toBytes()); } - @After + @AfterEach public void after() throws Exception { remoteTable.close(); } - @AfterClass + @AfterAll public static void tearDownAfterClass() throws Exception { REST_TEST_UTIL.shutdownServletContainer(); TEST_UTIL.shutdownMiniCluster(); @@ -707,5 +703,4 @@ public void testScanWithInlcudeStartStopRow() throws Exception { } assertEquals(5, counter); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java index 2618ff54180b..7c8f896b5248 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java @@ -17,37 +17,31 @@ */ package org.apache.hadoop.hbase.rest.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; import javax.xml.bind.UnmarshalException; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.rest.Constants; import org.apache.hadoop.hbase.rest.model.StorageClusterVersionModel; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.StringUtils; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Test class for {@link RemoteAdmin} to verify XML is parsed in a certain manner. */ -@Category(SmallTests.class) +@Tag(SmallTests.TAG) public class TestXmlParsing { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestXmlParsing.class); - private static final Logger LOG = LoggerFactory.getLogger(TestXmlParsing.class); @Test @@ -79,13 +73,14 @@ public void testFailOnExternalEntities() throws Exception { admin.getClusterVersion(); fail("Expected getClusterVersion() to throw an exception"); } catch (IOException e) { - assertEquals("Cause of exception ought to be a failure to parse the stream due to our " - + "invalid external entity. Make sure this isn't just a false positive due to " - + "implementation. see HBASE-19020.", UnmarshalException.class, e.getCause().getClass()); + assertEquals(UnmarshalException.class, e.getCause().getClass(), + "Cause of exception ought to be a failure to parse the stream due to our " + + "invalid external entity. Make sure this isn't just a false positive due to " + + "implementation. see HBASE-19020."); final String exceptionText = StringUtils.stringifyException(e); final String expectedText = "\"xee\""; - LOG.debug("exception text: '" + exceptionText + "'", e); - assertTrue("Exception does not contain expected text", exceptionText.contains(expectedText)); + LOG.debug("exception text: '{}'", exceptionText, e); + assertTrue(exceptionText.contains(expectedText), "Exception does not contain expected text"); } } } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java index 829091708573..766301b30e8f 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellModel.java @@ -17,28 +17,23 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestCellModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCellModel.class); - private static final long TIMESTAMP = 1245219839331L; private static final byte[] COLUMN = Bytes.toBytes("testcolumn"); private static final byte[] VALUE = Bytes.toBytes("testvalue"); @@ -71,16 +66,19 @@ protected void checkModel(CellModel model) { } @Override + @Test public void testBuildModel() throws Exception { checkModel(buildTestModel()); } @Override + @Test public void testFromXML() throws Exception { checkModel(fromXML(AS_XML)); } @Override + @Test public void testFromPB() throws Exception { checkModel(fromPB(AS_PB)); } @@ -93,16 +91,13 @@ public void testEquals() throws Exception { assertEquals(cellModel1, cellModel2); CellModel cellModel3 = new CellModel(); - assertFalse(cellModel1.equals(cellModel3)); + assertNotEquals(cellModel1, cellModel3); } @Test public void testToString() throws Exception { String expectedColumn = ToStringBuilder.reflectionToString(COLUMN, ToStringStyle.SIMPLE_STYLE); - CellModel cellModel = buildTestModel(); - System.out.println(cellModel); - assertTrue(StringUtils.contains(cellModel.toString(), expectedColumn)); } } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java index 38f0f43c0ceb..ffbb331ae1de 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestCellSetModel.java @@ -17,25 +17,21 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Iterator; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestCellSetModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestCellSetModel.class); - private static final byte[] ROW1 = Bytes.toBytes("testrow1"); private static final byte[] COLUMN1 = Bytes.toBytes("testcolumn1"); private static final byte[] VALUE1 = Bytes.toBytes("testvalue1"); @@ -121,18 +117,20 @@ protected void checkModel(CellSetModel model) { } @Override + @Test public void testBuildModel() throws Exception { checkModel(buildTestModel()); } @Override + @Test public void testFromXML() throws Exception { checkModel(fromXML(AS_XML)); } @Override + @Test public void testFromPB() throws Exception { checkModel(fromPB(AS_PB)); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java index 0001abe02d0b..45685b327cc4 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestColumnSchemaModel.java @@ -17,23 +17,18 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestColumnSchemaModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestColumnSchemaModel.class); - protected static final String COLUMN_NAME = "testcolumn"; protected static final boolean BLOCKCACHE = true; protected static final int BLOCKSIZE = 16384; @@ -70,14 +65,14 @@ protected ColumnSchemaModel buildTestModel() { @Override protected void checkModel(ColumnSchemaModel model) { - assertEquals("name", COLUMN_NAME, model.getName()); - assertEquals("block cache", BLOCKCACHE, model.__getBlockcache()); - assertEquals("block size", BLOCKSIZE, model.__getBlocksize()); - assertEquals("bloomfilter", BLOOMFILTER, model.__getBloomfilter()); - assertTrue("compression", model.__getCompression().equalsIgnoreCase(COMPRESSION)); - assertEquals("in memory", IN_MEMORY, model.__getInMemory()); - assertEquals("ttl", TTL, model.__getTTL()); - assertEquals("versions", VERSIONS, model.__getVersions()); + assertEquals(COLUMN_NAME, model.getName(), "name"); + assertEquals(BLOCKCACHE, model.__getBlockcache(), "block cache"); + assertEquals(BLOCKSIZE, model.__getBlocksize(), "block size"); + assertEquals(BLOOMFILTER, model.__getBloomfilter(), "bloomfilter"); + assertTrue(model.__getCompression().equalsIgnoreCase(COMPRESSION), "compression"); + assertEquals(IN_MEMORY, model.__getInMemory(), "in memory"); + assertEquals(TTL, model.__getTTL(), "ttl"); + assertEquals(VERSIONS, model.__getVersions(), "versions"); } @Override diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestModelBase.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestModelBase.java index 4cfe70e06399..5bc499aa76a9 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestModelBase.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestModelBase.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -29,7 +29,7 @@ import javax.xml.bind.JAXBException; import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.hadoop.hbase.rest.provider.JAXBContextResolver; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestNamespacesInstanceModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestNamespacesInstanceModel.java index 831a5642fb64..a9a01d120aeb 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestNamespacesInstanceModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestNamespacesInstanceModel.java @@ -17,25 +17,20 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; import java.util.HashMap; import java.util.Map; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestNamespacesInstanceModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestNamespacesInstanceModel.class); - public static final Map NAMESPACE_PROPERTIES = new HashMap<>(); public static final String NAMESPACE_NAME = "namespaceName"; diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestNamespacesModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestNamespacesModel.java index 30e0c44bcd86..c933605c6384 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestNamespacesModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestNamespacesModel.java @@ -17,25 +17,20 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; import java.util.List; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestNamespacesModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestNamespacesModel.class); - public static final String NAMESPACE_NAME_1 = "testNamespace1"; public static final String NAMESPACE_NAME_2 = "testNamespace2"; diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java index ad539e128481..bc1ac6b2328a 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java @@ -17,29 +17,25 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Iterator; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestRowModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRowModel.class); - private static final byte[] ROW1 = Bytes.toBytes("testrow1"); private static final byte[] COLUMN1 = Bytes.toBytes("testcolumn1"); private static final byte[] VALUE1 = Bytes.toBytes("testvalue1"); @@ -89,7 +85,7 @@ public void testEquals() throws Exception { assertEquals(rowModel1, rowModel2); RowModel rowModel3 = new RowModel(); - assertFalse(rowModel1.equals(rowModel3)); + assertNotEquals(rowModel1, rowModel3); } @Test @@ -97,8 +93,6 @@ public void testToString() throws Exception { String expectedRowKey = ToStringBuilder.reflectionToString(ROW1, ToStringStyle.SIMPLE_STYLE); RowModel rowModel = buildTestModel(); - System.out.println(rowModel); - assertTrue(StringUtils.contains(rowModel.toString(), expectedRowKey)); } } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java index b7b4da240eb8..c60308bba807 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java @@ -17,26 +17,23 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.rest.ScannerResultGenerator; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestScannerModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestScannerModel.class); private static final String PRIVATE = "private"; private static final String PUBLIC = "public"; @@ -106,7 +103,7 @@ protected void checkModel(ScannerModel model) { assertEquals(CACHE_BLOCKS, model.getCacheBlocks()); boolean foundLabel1 = false; boolean foundLabel2 = false; - if (model.getLabels() != null && model.getLabels().size() > 0) { + if (model.getLabels() != null && !model.getLabels().isEmpty()) { for (String label : model.getLabels()) { if (label.equals(PRIVATE)) { foundLabel1 = true; @@ -125,22 +122,22 @@ public void testExistingFilter() throws Exception { verifyException(CORRECT_FILTER); } - @Test(expected = IllegalArgumentException.class) + @Test public void testNonExistingFilter() throws Exception { final String UNKNOWN_FILTER = "{\"type\": \"UnknownFilter\", \"value\": \"cg==\"}"; - verifyException(UNKNOWN_FILTER); + assertThrows(IllegalArgumentException.class, () -> verifyException(UNKNOWN_FILTER)); } - @Test(expected = JsonMappingException.class) + @Test public void testIncorrectFilterThrowsJME() throws Exception { final String JME_FILTER = "{\"invalid_tag\": \"PrefixFilter\", \"value\": \"cg==\"}"; - verifyException(JME_FILTER); + assertThrows(JsonMappingException.class, () -> verifyException(JME_FILTER)); } - @Test(expected = JsonParseException.class) - public void tesIncorrecttFilterThrowsJPE() throws Exception { + @Test + public void tesIncorrectFilterThrowsJPE() throws Exception { final String JPE_FILTER = "{\"type\": \"PrefixFilter\",, \"value\": \"cg==\"}"; - verifyException(JPE_FILTER); + assertThrows(JsonParseException.class, () -> verifyException(JPE_FILTER)); } private void verifyException(final String FILTER) throws Exception { diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java index 22cc2230cc50..4a31beed3ef6 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterStatusModel.java @@ -17,24 +17,19 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.util.Iterator; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestStorageClusterStatusModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStorageClusterStatusModel.class); public TestStorageClusterStatusModel() throws Exception { super(StorageClusterStatusModel.class); @@ -106,7 +101,7 @@ protected void checkModel(StorageClusterStatusModel model) { assertEquals(1024, node.getMaxHeapSizeMB()); Iterator regions = node.getRegions().iterator(); StorageClusterStatusModel.Node.Region region = regions.next(); - assertTrue(Bytes.toString(region.getName()).equals("hbase:root,,0")); + assertEquals("hbase:root,,0", Bytes.toString(region.getName())); assertEquals(1, region.getStores()); assertEquals(1, region.getStorefiles()); assertEquals(0, region.getStorefileSizeMB()); @@ -127,7 +122,7 @@ protected void checkModel(StorageClusterStatusModel model) { assertEquals(1024, node.getMaxHeapSizeMB()); regions = node.getRegions().iterator(); region = regions.next(); - assertEquals(Bytes.toString(region.getName()), TableName.META_TABLE_NAME + ",,1246000043724"); + assertEquals(TableName.META_TABLE_NAME + ",,1246000043724", Bytes.toString(region.getName())); assertEquals(1, region.getStores()); assertEquals(1, region.getStorefiles()); assertEquals(0, region.getStorefileSizeMB()); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java index 6004cedcebcc..0004b005ffbb 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestStorageClusterVersionModel.java @@ -17,21 +17,17 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestStorageClusterVersionModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestStorageClusterVersionModel.class); - private static final String VERSION = "0.0.1-testing"; public TestStorageClusterVersionModel() throws Exception { @@ -55,6 +51,7 @@ protected void checkModel(StorageClusterVersionModel model) { } @Override + @Test public void testFromPB() throws Exception { // ignore test no pb } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java index a47cee53b182..d7897c99c379 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableInfoModel.java @@ -17,25 +17,21 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Iterator; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestTableInfoModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableInfoModel.class); - private static final String TABLE = "testtable"; private static final byte[] START_KEY = Bytes.toBytes("abracadbra"); private static final byte[] END_KEY = Bytes.toBytes("zzyzx"); @@ -80,18 +76,20 @@ protected void checkModel(TableInfoModel model) { } @Override + @Test public void testBuildModel() throws Exception { checkModel(buildTestModel()); } @Override + @Test public void testFromXML() throws Exception { checkModel(fromXML(AS_XML)); } @Override + @Test public void testFromPB() throws Exception { checkModel(fromPB(AS_PB)); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java index c034f0602dd1..9ee0569e4e04 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableListModel.java @@ -17,23 +17,18 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.util.Iterator; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestTableListModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableListModel.class); - private static final String TABLE1 = "table1"; private static final String TABLE2 = "table2"; private static final String TABLE3 = "table3"; diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java index 3a85b9ce0be6..a4f7f40a0e0e 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableRegionModel.java @@ -17,26 +17,21 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestTableRegionModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableRegionModel.class); - private static final String TABLE = "testtable"; private static final byte[] START_KEY = Bytes.toBytes("abracadbra"); private static final byte[] END_KEY = Bytes.toBytes("zzyzx"); @@ -59,8 +54,7 @@ public TestTableRegionModel() throws Exception { @Override protected TableRegionModel buildTestModel() { - TableRegionModel model = new TableRegionModel(TABLE, ID, START_KEY, END_KEY, LOCATION); - return model; + return new TableRegionModel(TABLE, ID, START_KEY, END_KEY, LOCATION); } @Override diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java index c12288e02093..bb9568dd1831 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestTableSchemaModel.java @@ -17,29 +17,20 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Iterator; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestTableSchemaModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestTableSchemaModel.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestTableSchemaModel.class); - public static final String TABLE_NAME = "testTable"; private static final boolean IS_META = false; private static final boolean IS_ROOT = false; @@ -117,5 +108,4 @@ public void testFromXML() throws Exception { public void testFromPB() throws Exception { checkModel(fromPB(AS_PB)); } - } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java index 8062e97e8e0f..65a0daa4ad2c 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestVersionModel.java @@ -17,21 +17,16 @@ */ package org.apache.hadoop.hbase.rest.model; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; -@Category({ RestTests.class, SmallTests.class }) +@Tag(RestTests.TAG) +@Tag(SmallTests.TAG) public class TestVersionModel extends TestModelBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestVersionModel.class); - private static final String REST_VERSION = "0.0.1"; private static final String OS_VERSION = "Linux 2.6.18-128.1.6.el5.centos.plusxen amd64"; private static final String JVM_VERSION = "Sun Microsystems Inc. 1.6.0_13-11.3-b02"; diff --git a/hbase-rpm/.blazar.yaml b/hbase-rpm/.blazar.yaml new file mode 100644 index 000000000000..a1bfcb2ae17b --- /dev/null +++ b/hbase-rpm/.blazar.yaml @@ -0,0 +1,30 @@ +buildpack: + name: Buildpack-RPMs + +env: + RPM_BUILD_COMMAND: ./build.sh + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +enableBuildTargets: + - almalinux9_amd64 + +depends: + - hbase + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +stepActivation: + uploadRpms: + branchRegexes: ['.*'] diff --git a/hbase-rpm/build.sh b/hbase-rpm/build.sh new file mode 100755 index 000000000000..b527ca732913 --- /dev/null +++ b/hbase-rpm/build.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e +set -x + +ROOT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +for iv in HBASE_VERSION SET_VERSION PKG_RELEASE; do + if [[ "X${!iv}" = "X" ]]; then + echo "Must specifiy $iv" + exit 1 + fi +done + +# Setup build dir +BUILD_DIR="${ROOT_DIR}/build" +rm -rf $BUILD_DIR +mkdir -p ${BUILD_DIR}/{SOURCES,SPECS,RPMS} +cp -a $ROOT_DIR/sources/* ${BUILD_DIR}/SOURCES/ +cp $ROOT_DIR/hbase.spec ${BUILD_DIR}/SPECS/ + +# Download bin tar built by hbase-assembly +SOURCES_DIR=$BUILD_DIR/SOURCES +mvn dependency:copy \ + -Dartifact=org.apache.hbase:hbase-assembly:${SET_VERSION}:tar.gz:bin \ + -DoutputDirectory=$SOURCES_DIR \ + -DlocalRepositoryDirectory=$SOURCES_DIR \ + -Dtransitive=false +INPUT_TAR=`ls -d $SOURCES_DIR/hbase-assembly-*.tar.gz` + +if [[ $HBASE_VERSION == *"-SNAPSHOT" ]]; then + # unreleased verion. do i want to denote that in the rpm release somehow? + # it can't be in the version, so strip here + HBASE_VERSION=${HBASE_VERSION//-SNAPSHOT/} +fi + +rpmbuild \ + --define "_topdir $BUILD_DIR" \ + --define "input_tar $INPUT_TAR" \ + --define "hbase_version ${HBASE_VERSION}" \ + --define "maven_version ${SET_VERSION}" \ + --define "release ${PKG_RELEASE}%{?dist}" \ + -bb \ + $BUILD_DIR/SPECS/hbase.spec + +if [[ -d $RPMS_OUTPUT_DIR ]]; then + mkdir -p $RPMS_OUTPUT_DIR + + # Move rpms to output dir for upload + + find ${BUILD_DIR}/RPMS -name "*.rpm" -exec mv {} $RPMS_OUTPUT_DIR/ \; +fi diff --git a/hbase-rpm/hbase.spec b/hbase-rpm/hbase.spec new file mode 100644 index 000000000000..107c92636f06 --- /dev/null +++ b/hbase-rpm/hbase.spec @@ -0,0 +1,131 @@ +# taken from hbase.spec in https://github.com/apache/bigtop/ +# greatly modified to simplify and fix dependencies to work in the hubspot environment + +%define hadoop_major_version 3.2 +%define hbase_major_version 2.4 +%define etc_hbase_conf %{_sysconfdir}/hbase/conf +%define etc_hbase_conf_dist %{etc_hbase_conf}.dist +%define hbase_home /usr/lib/hbase +%define bin_hbase %{hbase_home}/bin +%define lib_hbase %{hbase_home}/lib +%define conf_hbase %{hbase_home}/conf +%define logs_hbase %{hbase_home}/logs +%define pids_hbase %{hbase_home}/pids +%define man_dir %{_mandir} +%define hbase_username hbase +%define hadoop_home /usr/lib/hadoop +%define zookeeper_home /usr/lib/zookeeper + +# FIXME: brp-repack-jars uses unzip to expand jar files +# Unfortunately guice-2.0.jar pulled by ivy contains some files and directories without any read permission +# and make whole process to fail. +# So for now brp-repack-jars is being deactivated until this is fixed. +# See BIGTOP-294 +%define __os_install_post \ + %{_rpmconfigdir}/brp-compress ; \ + %{_rpmconfigdir}/brp-strip-static-archive %{__strip} ; \ + %{_rpmconfigdir}/brp-strip-comment-note %{__strip} %{__objdump} ; \ + /usr/lib/rpm/brp-python-bytecompile ; \ + %{nil} + +%define doc_hbase %{_docdir}/hbase-%{hbase_version} +%global initd_dir %{_sysconfdir}/rc.d/init.d +%define alternatives_cmd alternatives + +# Disable debuginfo package +%define debug_package %{nil} + +# HubSpot: use zstd because it decompresses much faster +%define _binary_payload w19.zstdio +%define _source_payload w19.zstdio + +Name: hbase +Version: %{hbase_version} +Release: %{release} +BuildArch: noarch +Summary: HBase is the Hadoop database. Use it when you need random, realtime read/write access to your Big Data. This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware. +URL: http://hbase.apache.org/ +Group: Systems/Daemons +Buildroot: %{_topdir}/INSTALL/hbase-%{maven_version} +License: ASL 2.0 +Source0: %{input_tar} +Source1: install_hbase.sh + +Requires: coreutils, /usr/sbin/useradd, /sbin/chkconfig, /sbin/service +Requires: hadoop >= %{hadoop_major_version} + +AutoReq: no + +%description +HBase is an open-source, distributed, column-oriented store modeled after Google' Bigtable: A Distributed Storage System for Structured Data by Chang et al. Just as Bigtable leverages the distributed data storage provided by the Google File System, HBase provides Bigtable-like capabilities on top of Hadoop. HBase includes: + + * Convenient base classes for backing Hadoop MapReduce jobs with HBase tables + * Query predicate push down via server side scan and get filters + * Optimizations for real time queries + * A high performance Thrift gateway + * A REST-ful Web service gateway that supports XML, Protobuf, and binary data encoding options + * Cascading source and sink modules + * Extensible jruby-based (JIRB) shell + * Support for exporting metrics via the Hadoop metrics subsystem to files or Ganglia; or via JMX + +%prep +%setup -n hbase-%{maven_version} + +%install +%__rm -rf $RPM_BUILD_ROOT +bash %{SOURCE1} \ + --input-tar=%{SOURCE0} \ + --doc-dir=%{doc_hbase} \ + --conf-dir=%{etc_hbase_conf_dist} \ + --prefix=$RPM_BUILD_ROOT + +%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/log/hbase +ln -s %{_localstatedir}/log/hbase %{buildroot}/%{logs_hbase} + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/run/hbase +ln -s %{_localstatedir}/run/hbase %{buildroot}/%{pids_hbase} + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/lib/hbase + +%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin + +# Pull hadoop from its packages +rm -f $RPM_BUILD_ROOT/%{lib_hbase}/{hadoop,slf4j-log4j12-}*.jar + +ln -f -s %{hadoop_home}/client/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-hdfs-client.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-common.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-jobclient.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-client.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_hbase} + +%pre +getent group hbase 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hbase +getent passwd hbase 2>&1 > /dev/null || /usr/sbin/useradd -c "HBase" -s /sbin/nologin -g hbase -r -d /var/lib/hbase hbase 2> /dev/null || : + +%post +%{alternatives_cmd} --install %{etc_hbase_conf} %{name}-conf %{etc_hbase_conf_dist} 30 + +%files +%defattr(-,hbase,hbase) +%{logs_hbase} +%{pids_hbase} +%dir %{_localstatedir}/log/hbase +%dir %{_localstatedir}/run/hbase +%dir %{_localstatedir}/lib/hbase + +%defattr(-,root,root) +%{hbase_home} +%{hbase_home}/hbase-*.jar +/usr/bin/hbase +%config(noreplace) %{etc_hbase_conf_dist} + +# files from doc package +%defattr(-,root,root) +%doc %{doc_hbase}/ diff --git a/hbase-rpm/sources/hbase.1 b/hbase-rpm/sources/hbase.1 new file mode 100644 index 000000000000..349218fe1d87 --- /dev/null +++ b/hbase-rpm/sources/hbase.1 @@ -0,0 +1,88 @@ +.\" Licensed to the Apache Software Foundation (ASF) under one or more +.\" contributor license agreements. See the NOTICE file distributed with +.\" this work for additional information regarding copyright ownership. +.\" The ASF licenses this file to You under the Apache License, Version 2.0 +.\" (the "License"); you may not use this file except in compliance with +.\" the License. You may obtain a copy of the License at +.\" +.\" http://www.apache.org/licenses/LICENSE-2.0 +.\" +.\" Unless required by applicable law or agreed to in writing, software +.\" distributed under the License is distributed on an "AS IS" BASIS, +.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.\" See the License for the specific language governing permissions and +.\" limitations under the License. +.\" +.\" Process this file with +.\" groff -man -Tascii hbase.1 +.\" +.TH hbase 1 "October 2010 " Linux "User Manuals" + +.SH NAME +HBase \- HBase is the Hadoop database. + +.SH SYNOPSIS + +.B hbase +\fICOMMAND\fR + +.SH DESCRIPTION + +HBase is the Hadoop database. Use it when you need random, realtime +read/write access to your Big Data. This project's goal is the hosting +of very large tables -- billions of rows X millions of columns -- atop +clusters of commodity hardware. + +HBase is an open-source, distributed, versioned, column-oriented store +modeled after Google's Bigtable: A Distributed Storage System for +Structured Data by Chang et al. Just as Bigtable leverages the +distributed data storage provided by the Google File System, HBase +provides Bigtable-like capabilities on top of Hadoop. + +For more information about HBase, see http://hbase.apache.org. + +\fICOMMAND\fR may be one of the following: + shell run the HBase shell + shell-tests run the HBase shell tests + zkcli run the ZooKeeper shell + master run an HBase HMaster node + regionserver run an HBase HRegionServer node + zookeeper run a Zookeeper server + rest run an HBase REST server + thrift run an HBase Thrift server + avro run an HBase Avro server + migrate upgrade an hbase.rootdir + hbck run the hbase 'fsck' tool + or + CLASSNAME run the class named CLASSNAME + +Most commands print help when invoked w/o parameters or with --help. + +.SH ENVIRONMENT + +.IP JAVA_HOME +The java implementation to use. Overrides JAVA_HOME. + +.IP HBASE_CLASSPATH +Extra Java CLASSPATH entries. + +.IP HBASE_HEAPSIZE +The maximum amount of heap to use, in MB. Default is 1000. + +.IP HBASE_OPTS +Extra Java runtime options. + +.IP HBASE_CONF_DIR +Alternate conf dir. Default is ${HBASE_HOME}/conf. + +.IP HBASE_ROOT_LOGGER +The root appender. Default is INFO,console + +.IP HIVE_OPT +Extra Java runtime options. + +.IP HADOOP_HOME +Optionally, the Hadoop home to run with. + +.SH COPYRIGHT +Copyright (C) 2010 The Apache Software Foundation. All rights reserved. diff --git a/hbase-rpm/sources/install_hbase.sh b/hbase-rpm/sources/install_hbase.sh new file mode 100755 index 000000000000..95265d2100c8 --- /dev/null +++ b/hbase-rpm/sources/install_hbase.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +usage() { + echo " +usage: $0 + Required not-so-options: + --mvn-target-dir=DIR path to the output of the mvn assembly + --prefix=PREFIX path to install into + + Optional options: + --doc-dir=DIR path to install docs into [/usr/share/doc/hbase] + --lib-dir=DIR path to install hbase home [/usr/lib/hbase] + --installed-lib-dir=DIR path where lib-dir will end up on target system + --bin-dir=DIR path to install bins [/usr/bin] + --examples-dir=DIR path to install examples [doc-dir/examples] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'doc-dir:' \ + -l 'lib-dir:' \ + -l 'installed-lib-dir:' \ + -l 'bin-dir:' \ + -l 'examples-dir:' \ + -l 'conf-dir:' \ + -l 'input-tar:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --input-tar) + INPUT_TAR=$2 ; shift 2 + ;; + --doc-dir) + DOC_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --examples-dir) + EXAMPLES_DIR=$2 ; shift 2 + ;; + --conf-dir) + CONF_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX INPUT_TAR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +MAN_DIR=${MAN_DIR:-/usr/share/man/man1} +DOC_DIR=${DOC_DIR:-/usr/share/doc/hbase} +LIB_DIR=${LIB_DIR:-/usr/lib/hbase} + +BIN_DIR=${BIN_DIR:-/usr/lib/hbase/bin} +ETC_DIR=${ETC_DIR:-/etc/hbase} +CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist} +THRIFT_DIR=${THRIFT_DIR:-${LIB_DIR}/include/thrift} + +EXTRACT_DIR=extracted +rm -rf $EXTRACT_DIR +mkdir $EXTRACT_DIR + +version_part=$SET_VERSION +if [ -z "$version_part" ]; then + version_part=$HBASE_VERSION +fi + +tar -C $EXTRACT_DIR --strip-components=1 -xzf $INPUT_TAR + +# we do not need the shaded clients in our rpm. they bloat the size and cause classpath issues for hbck2. +rm -rf $EXTRACT_DIR/lib/shaded-clients + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/lib +install -d -m 0755 $PREFIX/$DOC_DIR +install -d -m 0755 $PREFIX/$BIN_DIR +install -d -m 0755 $PREFIX/$ETC_DIR +install -d -m 0755 $PREFIX/$MAN_DIR +install -d -m 0755 $PREFIX/$THRIFT_DIR + +cp -ra $EXTRACT_DIR/lib/* ${PREFIX}/${LIB_DIR}/lib/ +cp $EXTRACT_DIR/lib/hbase*.jar $PREFIX/$LIB_DIR + +# We do not currently run "mvn site", so do not have a docs dir. +# Only copy contents if dir exists +if [ -n "$(ls -A $EXTRACT_DIR/docs 2>/dev/null)" ]; then + cp -a $EXTRACT_DIR/docs/* $PREFIX/$DOC_DIR + cp $EXTRACT_DIR/*.txt $PREFIX/$DOC_DIR/ +else + echo "Doc generation is currently disabled in our RPM build. If this is an issue, it should be possible to enable them with some work. See https://git.hubteam.com/HubSpot/apache-hbase/blob/hubspot-2/rpm/sources/do-component-build#L17-L24 for details." > $PREFIX/$DOC_DIR/README.txt +fi + +cp -a $EXTRACT_DIR/conf $PREFIX/$CONF_DIR +cp -a $EXTRACT_DIR/bin/* $PREFIX/$BIN_DIR + +# Purge scripts that don't work with packages +for file in rolling-restart.sh graceful_stop.sh local-regionservers.sh \ + master-backup.sh regionservers.sh zookeepers.sh hbase-daemons.sh \ + start-hbase.sh stop-hbase.sh local-master-backup.sh ; do + rm -f $PREFIX/$BIN_DIR/$file +done + + +ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/conf + +# Make a symlink of hbase.jar to hbase-version.jar +pushd `pwd` +cd $PREFIX/$LIB_DIR +for i in `ls hbase*jar | grep -v tests.jar` +do + ln -s $i `echo $i | sed -n 's/\(.*\)\(-[0-9].*\)\(.jar\)/\1\3/p'` +done +popd + +wrapper=$PREFIX/usr/bin/hbase +mkdir -p `dirname $wrapper` +cat > $wrapper <test - junit - junit + org.junit.jupiter + junit-jupiter-api test + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine + @@ -175,11 +189,6 @@ true - - - org.apache.maven.plugins - maven-source-plugin - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java index 9d1b03e18eba..2a97668e0f97 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedSet; import java.util.concurrent.Future; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -47,7 +48,6 @@ import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.TableStateManager; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.assignment.RegionStateNode; import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; import org.apache.hadoop.hbase.net.Address; @@ -532,8 +532,7 @@ public BalanceResponse balanceRSGroup(String groupName, BalanceRequest request) Map groupRIT = rsGroupGetRegionsInTransition(groupName); if (groupRIT.size() > 0 && !request.isIgnoreRegionsInTransition()) { LOG.debug("Not running balancer because {} region(s) in transition: {}", groupRIT.size(), - StringUtils.abbreviate( - master.getAssignmentManager().getRegionStates().getRegionsInTransition().toString(), + StringUtils.abbreviate(master.getAssignmentManager().getRegionsInTransition().toString(), 256)); return responseBuilder.build(); } @@ -654,14 +653,12 @@ public void updateConfiguration(String groupName) throws IOException { private Map rsGroupGetRegionsInTransition(String groupName) throws IOException { + SortedSet tablesInGroup = getRSGroupInfo(groupName).getTables(); Map rit = Maps.newTreeMap(); - AssignmentManager am = master.getAssignmentManager(); - for (TableName tableName : getRSGroupInfo(groupName).getTables()) { - for (RegionInfo regionInfo : am.getRegionStates().getRegionsOfTable(tableName)) { - RegionState state = am.getRegionStates().getRegionTransitionState(regionInfo); - if (state != null) { - rit.put(regionInfo.getEncodedName(), state); - } + for (RegionStateNode regionNode : master.getAssignmentManager().getRegionsInTransition()) { + TableName tn = regionNode.getTable(); + if (tablesInGroup.contains(tn)) { + rit.put(regionNode.getRegionInfo().getEncodedName(), regionNode.toRegionState()); } } return rit; diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java index e8cc70d27858..bb6bc01c2fd3 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java @@ -30,14 +30,17 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.compaction.TestMajorCompactorTTL; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +@Category(LargeTests.class) public class TestRSGroupMajorCompactionTTL extends TestMajorCompactorTTL { @ClassRule public static final HBaseClassTestRule CLASS_RULE = diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java index dee907fea623..12cc5df1cab1 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java @@ -101,16 +101,13 @@ public void testFallback() throws Exception { assertRegionsInGroup(tableName, FALLBACK_GROUP); // add a new server to default group, regions move to default group - JVMClusterUtil.RegionServerThread t = - TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); - Address startRSAddress = t.getRegionServer().getServerName().getAddress(); - TEST_UTIL.waitFor(3000, - () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).containsServer(startRSAddress)); + TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); assertTrue(master.balance().isBalancerRan()); assertRegionsInGroup(tableName, RSGroupInfo.DEFAULT_GROUP); // add a new server to test group, regions move back - t = TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); + JVMClusterUtil.RegionServerThread t = + TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); rsGroupAdmin.moveServers( Collections.singleton(t.getRegionServer().getServerName().getAddress()), groupName); assertTrue(master.balance().isBalancerRan()); @@ -119,11 +116,11 @@ public void testFallback() throws Exception { TEST_UTIL.deleteTable(tableName); } - private void assertRegionsInGroup(TableName tableName, String group) throws IOException { + private void assertRegionsInGroup(TableName table, String group) throws IOException { ProcedureTestingUtility .waitAllProcedures(TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor()); RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(group); - master.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName).forEach(region -> { + master.getAssignmentManager().getRegionStates().getRegionsOfTable(table).forEach(region -> { Address regionOnServer = master.getAssignmentManager().getRegionStates() .getRegionAssignments().get(region).getAddress(); assertTrue(groupInfo.getServers().contains(regionOnServer)); diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java index e11d8f5717cf..3f82362a1ba0 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue; import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -46,7 +45,6 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hbase.util.VersionInfo; import org.junit.After; import org.junit.AfterClass; @@ -60,7 +58,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category({ LargeTests.class }) +@Category(LargeTests.class) public class TestRSGroupsKillRS extends TestRSGroupsBase { @ClassRule @@ -266,24 +264,27 @@ public void testLowerMetaGroupVersion() throws Exception { Address address = servers.iterator().next(); int majorVersion = VersionInfo.getMajorVersion(originVersion); assertTrue(majorVersion >= 1); - String lowerVersion = String.valueOf(majorVersion - 1) + originVersion.split("\\.")[1]; - setFinalStatic(Version.class.getField("version"), lowerVersion); - TEST_UTIL.getMiniHBaseCluster().startRegionServer(address.getHostName(), address.getPort()); - assertEquals(NUM_SLAVES_BASE, - TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size()); - assertTrue(VersionInfo.compareVersion(originVersion, - master.getRegionServerVersion(getServerName(servers.iterator().next()))) > 0); - LOG.debug("wait for META assigned..."); - // SCP finished, which means all regions assigned too. - TEST_UTIL.waitFor(60000, () -> !TEST_UTIL.getHBaseCluster().getMaster().getProcedures().stream() - .filter(p -> (p instanceof ServerCrashProcedure)).findAny().isPresent()); + String lowerVersion = + String.valueOf(majorVersion - 1) + originVersion.substring(originVersion.indexOf(".")); + try { + setVersionInfoVersion(lowerVersion); + TEST_UTIL.getMiniHBaseCluster().startRegionServer(address.getHostName(), address.getPort()); + assertEquals(NUM_SLAVES_BASE, + TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size()); + assertTrue(VersionInfo.compareVersion(originVersion, + master.getRegionServerVersion(getServerName(servers.iterator().next()))) > 0); + LOG.debug("wait for META assigned..."); + // SCP finished, which means all regions assigned too. + TEST_UTIL.waitFor(60000, () -> !TEST_UTIL.getHBaseCluster().getMaster().getProcedures() + .stream().filter(p -> (p instanceof ServerCrashProcedure)).findAny().isPresent()); + } finally { + setVersionInfoVersion(Version.version); + } } - private static void setFinalStatic(Field field, Object newValue) throws Exception { - field.setAccessible(true); - Field modifiersField = ReflectionUtils.getModifiersField(); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - field.set(null, newValue); + private static void setVersionInfoVersion(String newValue) throws Exception { + Field f = VersionInfo.class.getDeclaredField("version"); + f.setAccessible(true); + f.set(null, newValue); } } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java index 452fd63d3c88..0d1810101a38 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java @@ -138,7 +138,7 @@ public boolean evaluate() throws Exception { @Override public boolean evaluate() throws Exception { return groupRS.getNumberOfOnlineRegions() < 1 - && master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() < 1; + && master.getAssignmentManager().getRegionsInTransitionCount() < 1; } }); // Move table to group and wait. diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 5b7a6aa09ed6..70f4afcecde8 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -197,6 +197,12 @@ compile true + + com.github.stephenc.jcip + jcip-annotations + compile + true + com.github.ben-manes.caffeine caffeine @@ -234,10 +240,6 @@ org.apache.zookeeper zookeeper - - org.jamon - jamon-runtime - javax.servlet javax.servlet-api @@ -293,8 +295,28 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility test @@ -302,6 +324,21 @@ mockito-core test + + org.mockito + mockito-inline + test + + + + org.mockito + mockito-junit-jupiter + test + org.slf4j jcl-over-slf4j @@ -409,13 +446,13 @@ false - ${build.year} - ${license.debug.print.included} - ${license.bundles.dependencies} - ${license.bundles.jquery} - ${license.bundles.vega} - ${license.bundles.logo} - ${license.bundles.bootstrap} + ${build.year} + ${license.debug.print.included} + ${license.bundles.dependencies} + ${license.bundles.jquery} + ${license.bundles.vega} + ${license.bundles.logo} + ${license.bundles.bootstrap} ${project.groupId}:hbase-resource-bundle:${project.version} @@ -471,7 +508,6 @@ generate-sources - ${project.build.directory}/generated-jamon ${project.build.directory}/generated-sources/java @@ -556,33 +592,33 @@ + + + + + + + + + + + + + - + - + + + - - org.jamon - jamon-maven-plugin - - - - translate - - generate-sources - - src/main/jamon - target/generated-jamon - - - - maven-surefire-plugin @@ -998,6 +1034,69 @@ hadoop-minikdc test + + + org.apache.directory.server + apacheds-core + ${apacheds.version} + test + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcpkix-jdk15on + + + + + org.apache.directory.server + apacheds-protocol-ldap + ${apacheds.version} + test + + + org.bouncycastle + bcprov-jdk15on + + + + + org.apache.directory.server + apacheds-ldif-partition + ${apacheds.version} + test + + + org.apache.directory.api + api-ldap-codec-core + ${ldap-api.version} + test + + + org.apache.directory.api + api-ldap-model + ${ldap-api.version} + test + + + org.apache.directory.server + apacheds-server-integ + ${apacheds.version} + test + + + log4j + log4j + + + @@ -1044,22 +1143,6 @@ - - - org.jamon - jamon-maven-plugin - [2.3.4,) - - translate - - - - - false - true - - - org.apache.maven.plugins @@ -1095,31 +1178,6 @@ - - - org.apache.maven.plugins - maven-eclipse-plugin - - - org.jamon.project.jamonnature - - - org.jamon.project.templateBuilder - org.eclipse.jdt.core.javabuilder - org.jamon.project.markerUpdater - - - - .settings/org.jamon.prefs - # now - eclipse.preferences.version=1 - templateSourceDir=src/main/jamon - templateOutputDir=target/generated-jamon - - - - - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon deleted file mode 100644 index 8798e930c707..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/common/TaskMonitorTmpl.jamon +++ /dev/null @@ -1,140 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%import> -java.util.*; -org.apache.hadoop.hbase.monitoring.*; -org.apache.hadoop.util.StringUtils; - -<%args> -TaskMonitor taskMonitor = TaskMonitor.get(); -String filter = "general"; -String format = "html"; -String parent = ""; - - -<%if format.equals("json")%> - <& renderTasks; filter=filter &> -<%else> -

Tasks

- -
- -
-
- <& jsonView; filter="all" &> - <& renderTasks; filter="all" &> -
-
- <& jsonView; filter="general" &> - <& renderTasks; filter="general" &> -
-
- <& jsonView; filter="handler" &> - <& renderTasks; filter="handler" &> -
-
- <& jsonView; filter="rpc" &> - <& renderTasks; filter="rpc" &> -
-
- <& jsonView; filter="operation" &> - <& renderTasks; filter="operation" &> -
-
-
- - -<%def jsonView> - <%args> - String filter; - - View as JSON - - -<%def renderTasks> - <%args> - String filter; - - <%java> - List tasks = taskMonitor.getTasks(filter); - long now = System.currentTimeMillis(); - Collections.sort(tasks, (t1, t2) -> Long.compare(t1.getStateTime(), t2.getStateTime())); - boolean first = true; - - <%if format.equals("json") %> - [<%for MonitoredTask task : tasks%><%if first%><%java>first = false;<%else>,<% task.toJSON() %>] - <%else> - <%if tasks.isEmpty()%> -

No tasks currently running on this node.

- <%else> - - - - - - - - - <%for MonitoredTask task : tasks %> - - - - - - - - -
Start TimeDescriptionStateStatusCompletion Time
<% new Date(task.getStartTime()) %><% task.getDescription() %><% task.getState() %> - (since <% StringUtils.formatTimeDiff(now, task.getStateTime()) %> ago) - <% task.getStatus() %> - (since <% StringUtils.formatTimeDiff(now, task.getStatusTime()) %> - ago) - <%if task.getCompletionTimestamp() < 0 %> - <% task.getState() %> - <%else> - <% new Date(task.getCompletionTimestamp()) %> (since <% StringUtils.formatTimeDiff(now, task.getCompletionTimestamp()) %> ago) - -
- - - - -<%def stateCss> - <%args> - MonitoredTask.State state; - -<%java> if (state == MonitoredTask.State.COMPLETE) { alert alert-success<%java> } -<%java> else if (state == MonitoredTask.State.ABORTED) { alert alert-danger<%java> } - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon deleted file mode 100644 index ee899a7340dc..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon +++ /dev/null @@ -1,128 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%import> -java.util.Map; -java.util.Set; -java.util.SortedSet; -java.util.concurrent.atomic.AtomicInteger; -java.util.stream.Collectors; -org.apache.hadoop.conf.Configuration; -org.apache.hadoop.hbase.HBaseConfiguration; -org.apache.hadoop.hbase.HConstants; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.client.RegionInfo; -org.apache.hadoop.hbase.client.RegionInfoDisplay; -org.apache.hadoop.hbase.master.RegionState; -org.apache.hadoop.hbase.master.assignment.AssignmentManager; -org.apache.hadoop.hbase.master.assignment.AssignmentManager.RegionInTransitionStat; -org.apache.hadoop.hbase.master.assignment.RegionStates.RegionFailedOpen; -org.apache.hadoop.hbase.util.Pair; - -<%args> -AssignmentManager assignmentManager; -int limit = 100; - - -<%java> -SortedSet rit = assignmentManager.getRegionStates() - .getRegionsInTransitionOrderedByTimestamp(); - - -<%if !rit.isEmpty() %> -<%java> -long currentTime = System.currentTimeMillis(); -RegionInTransitionStat ritStat = assignmentManager.computeRegionInTransitionStat(); - -int numOfRITs = rit.size(); -int ritsPerPage = Math.min(5, numOfRITs); -int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / ritsPerPage); - -
-

Regions in Transition

-

<% numOfRITs %> region(s) in transition. - <%if ritStat.hasRegionsTwiceOverThreshold() %> - - <%elseif ritStat.hasRegionsOverThreshold() %> - - <%else> - - - <% ritStat.getTotalRITsOverThreshold() %> region(s) in transition for - more than <% ritStat.getRITThreshold() %> milliseconds. - -

-
-
- <%java int recordItr = 0; %> - <%for RegionState rs : rit %> - <%if (recordItr % ritsPerPage) == 0 %> - <%if recordItr == 0 %> -
- <%else> -
- - - - - - <%if ritStat.isRegionTwiceOverThreshold(rs.getRegion()) %> - - <%elseif ritStat.isRegionOverThreshold(rs.getRegion()) %> - - <%else> - - - <%java> - String retryStatus = "0"; - RegionFailedOpen regionFailedOpen = assignmentManager - .getRegionStates().getFailedOpen(rs.getRegion()); - if (regionFailedOpen != null) { - retryStatus = Integer.toString(regionFailedOpen.getRetries()); - } else if (rs.getState() == RegionState.State.FAILED_OPEN) { - retryStatus = "Failed"; - } - - - - - - <%java recordItr++; %> - <%if (recordItr % ritsPerPage) == 0 %> -
RegionStateRIT time (ms) Retries
<% rs.getRegion().getEncodedName() %> - <% RegionInfoDisplay.getDescriptiveNameFromRegionStateForDisplay(rs, - assignmentManager.getConfiguration()) %><% (currentTime - rs.getStamp()) %> <% retryStatus %>
-
- - - - <%if (recordItr % ritsPerPage) != 0 %> - <%for ; (recordItr % ritsPerPage) != 0 ; recordItr++ %> - - - -
- -
- - - -
-
- - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon deleted file mode 100644 index 21af264bbe34..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon +++ /dev/null @@ -1,70 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%args> -HMaster master; - -<%import> -java.util.*; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.ClusterMetrics; -org.apache.hadoop.hbase.master.HMaster; -org.apache.hbase.thirdparty.com.google.common.base.Preconditions; - -<%if (!master.isActiveMaster()) %> - <%java> - ServerName active_master = master.getActiveMaster().orElse(null); - Preconditions.checkState(active_master != null, "Failed to retrieve active master's ServerName!"); - int activeInfoPort = master.getActiveMasterInfoPort(); - -
-
-

Backup Master <% master.getServerName().getHostname() %>

-
-
-

Current Active Master: <% active_master.getHostname() %>

-<%else> -

Backup Masters

- - - - - - - - <%java> - Collection backup_masters = master.getBackupMasters(); - ServerName [] backupServerNames = backup_masters.toArray(new ServerName[backup_masters.size()]); - Arrays.sort(backupServerNames); - for (ServerName serverName : backupServerNames) { - int infoPort = master.getBackupMasterInfoPort(serverName); - - - - - - - <%java> - } - - -
ServerNamePortStart Time
<% serverName.getHostname() %> - <% serverName.getPort() %><% new Date(serverName.getStartcode()) %>
Total:<% backupServerNames.length %>
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon deleted file mode 100644 index d1b3f8719d2e..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ /dev/null @@ -1,808 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%args> -HMaster master; -Map frags = null; -ServerName metaLocation = null; -List servers = null; -Set deadServers = null; -boolean catalogJanitorEnabled = true; -String filter = "general"; -String format = "html"; -ServerManager serverManager = null; -AssignmentManager assignmentManager = null; - -<%import> -java.util.*; -java.net.URLEncoder; -java.io.IOException; -org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil; -org.apache.hadoop.hbase.client.RegionInfo; -org.apache.hadoop.hbase.client.TableDescriptor; -org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -org.apache.hadoop.hbase.replication.ReplicationPeerDescription; -org.apache.hadoop.hbase.HBaseConfiguration; -org.apache.hadoop.hbase.HConstants; -org.apache.hadoop.hbase.HTableDescriptor; -org.apache.hadoop.hbase.NamespaceDescriptor; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.TableName; -org.apache.hadoop.hbase.RSGroupTableAccessor; -org.apache.hadoop.hbase.client.Admin; -org.apache.hadoop.hbase.client.MasterSwitchType; -org.apache.hadoop.hbase.client.TableState; -org.apache.hadoop.hbase.master.assignment.AssignmentManager; -org.apache.hadoop.hbase.master.DeadServer; -org.apache.hadoop.hbase.master.HMaster; -org.apache.hadoop.hbase.master.RegionState; -org.apache.hadoop.hbase.master.ServerManager; -org.apache.hadoop.hbase.rsgroup.RSGroupInfo; -org.apache.hadoop.hbase.protobuf.ProtobufUtil; -org.apache.hadoop.hbase.quotas.QuotaUtil; -org.apache.hadoop.hbase.security.access.PermissionStorage; -org.apache.hadoop.hbase.security.visibility.VisibilityConstants; -org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; -org.apache.hadoop.hbase.tool.CanaryTool; -org.apache.hadoop.hbase.util.Bytes; -org.apache.hadoop.hbase.util.CommonFSUtils; -org.apache.hadoop.hbase.util.JvmVersion; -org.apache.hadoop.hbase.util.PrettyPrinter; -org.apache.hadoop.util.StringUtils; -org.apache.hadoop.hbase.util.Strings; - - -<%if format.equals("json") %> - <& ../common/TaskMonitorTmpl; filter = filter; format = "json" &> - <%java return; %> - -<%java> -ServerManager serverManager = master.getServerManager(); -AssignmentManager assignmentManager = master.getAssignmentManager(); - - -<%class> - public String formatZKString() { - StringBuilder quorums = new StringBuilder(); - String zkQuorum = master.getZooKeeper().getQuorum(); - - if (null == zkQuorum) { - return quorums.toString(); - } - - String[] zks = zkQuorum.split(","); - - if (zks.length == 0) { - return quorums.toString(); - } - - for(int i = 0; i < zks.length; ++i) { - quorums.append(zks[i].trim()); - - if (i != (zks.length - 1)) { - quorums.append("
"); - } - } - - return quorums.toString(); - } - - -<%class> - public static String getUserTables(HMaster master, List tables){ - if (master.isInitialized()){ - try { - Map descriptorMap = master.getTableDescriptors().getAll(); - if (descriptorMap != null) { - for (TableDescriptor desc : descriptorMap.values()) { - if (!desc.getTableName().isSystemTable()) { - tables.add(desc); - } - } - } - } catch (IOException e) { - return "Got user tables error, " + e.getMessage(); - } - } - return null; - } - - - - - - - - <%if master.isActiveMaster() %>Master: <%else>Backup Master: </%if> - <% master.getServerName().getHostname() %> - - - - - - - - - - - -
- <%if master.isActiveMaster() %> -
-
-

Master <% master.getServerName().getHostname() %>

-
-
- -
- - <%if JvmVersion.isBadJvmVersion() %> - - - <%if master.isInitialized() && !catalogJanitorEnabled %> - - - <%if master.isInMaintenanceMode() %> - - - <%if !master.isBalancerOn() %> - - - <%if !master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) %> - - - <%if !master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) %> - - - <%if master.getAssignmentManager() != null %> - <& AssignmentManagerStatusTmpl; assignmentManager=master.getAssignmentManager()&> - - <%if !master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null %> - <%if master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null && - serverManager.getOnlineServersList().size() > 0 %> -
-

RSGroup

- <& RSGroupListTmpl; master= master; serverManager= serverManager&> -
- - -
-
-
-

Region Servers

- <& RegionServerListTmpl; master= master; servers = servers &> - - <%if (deadServers != null) %> - <& deadRegionServers &> - -
-
-
-
- <& BackupMasterStatusTmpl; master = master &> -
-
-
-
-

Tables

-
- -
-
- <%if (metaLocation != null) %> - <& userTables &> - -
-
- <%if (metaLocation != null) %> - <& catalogTables &> - -
-
-
-
-
-
-
-
-
-

Region Visualizer

- <& RegionVisualizerTmpl &> -
-
-
-
-

Peers

- <& peerConfigs &> -
-
- <%else> -
- <& BackupMasterStatusTmpl; master = master &> -
- - - -
- <& ../common/TaskMonitorTmpl; filter = filter; parent = "/master-status" &> -
- -
-

Software Attributes

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <%escape #n> - - - - - - - - - - - - - - - - - - - - - - - - <%if master.isActiveMaster() %> - - - - - - - - - - - - - - - - <%if frags != null %> - - - - - - - - - - - - - - - - - -
Attribute NameValueDescription
JVM Version<% JvmVersion.getVersion() %>JVM vendor and version
HBase Version<% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision
HBase Compiled<% org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <% org.apache.hadoop.hbase.util.VersionInfo.getUser() %>When HBase version was compiled and by whom
HBase Source Checksum<% org.apache.hadoop.hbase.util.VersionInfo.getSrcChecksum() %>HBase source SHA512 checksum
Hadoop Version<% org.apache.hadoop.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.util.VersionInfo.getRevision() %>Hadoop version and revision
Hadoop Compiled<% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %>When Hadoop version was compiled and by whom
Hadoop Source Checksum<% org.apache.hadoop.util.VersionInfo.getSrcChecksum() %>Hadoop source MD5 checksum
ZooKeeper Client Version<% org.apache.zookeeper.Version.getVersion() %>, revision=<% org.apache.zookeeper.Version.getRevisionHash() %>ZooKeeper client version and revision hash
ZooKeeper Client Compiled<% org.apache.zookeeper.Version.getBuildDate() %>When ZooKeeper client version was compiled
ZooKeeper Quorum <% formatZKString() %> Addresses of all registered ZK servers. For more, see zk dump.
ZooKeeper Base Path <% master.getZooKeeper().getZNodePaths().baseZNode %>Root node of this cluster in ZK.
Cluster Key <% formatZKString() %>:<% master.getZooKeeper().getZNodePaths().baseZNode %>Key to add this cluster as a peer for replication. Use 'help "add_peer"' in the shell for details.
HBase Root Directory<% CommonFSUtils.getRootDir(master.getConfiguration()).toString() %>Location of HBase home directory
HMaster Start Time<% new Date(master.getMasterStartTime()) %>Date stamp of when this HMaster was started
HMaster Active Time<% new Date(master.getMasterActiveTime()) %>Date stamp of when this HMaster became active
HBase Cluster ID<% master.getClusterId() != null ? master.getClusterId() : "Not set" %>Unique identifier generated for each HBase cluster
Load average<% master.getServerManager() == null ? "0.00" : - StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %>Average number of regions per regionserver. Naive computation.
Fragmentation<% frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %>Overall fragmentation of all tables, including hbase:meta
Coprocessors<% master.getMasterCoprocessorHost() == null ? "[]" : - java.util.Arrays.toString(master.getMasterCoprocessors()) %>Coprocessors currently loaded by the master
LoadBalancer<% master.getLoadBalancerClassName() %>LoadBalancer to be used in the Master
-
-
- - - - - - - - - - - - -<%def catalogTables> -<%java> - List sysTables = master.isInitialized() ? - master.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR) : null; - -<%if (sysTables != null && sysTables.size() > 0)%> - - - - <%if (frags != null) %> - - - - -<%for TableDescriptor systemTable : sysTables%> - -<%java>TableName tableName = systemTable.getTableName(); - - <%if (frags != null)%> - - - <%java>String description = null; - if (tableName.equals(TableName.META_TABLE_NAME)){ - description = "The hbase:meta table holds references to all User Table regions."; - } else if (tableName.equals(CanaryTool.DEFAULT_WRITE_TABLE_NAME)){ - description = "The hbase:canary table is used to sniff the write availbility of" - + " each regionserver."; - } else if (tableName.equals(PermissionStorage.ACL_TABLE_NAME)){ - description = "The hbase:acl table holds information about acl."; - } else if (tableName.equals(VisibilityConstants.LABELS_TABLE_NAME)){ - description = "The hbase:labels table holds information about visibility labels."; - } else if (tableName.equals(TableName.NAMESPACE_TABLE_NAME)){ - description = "The hbase:namespace table holds information about namespaces."; - } else if (tableName.equals(QuotaUtil.QUOTA_TABLE_NAME)){ - description = "The hbase:quota table holds quota information about number" + - " or size of requests in a given time frame."; - } else if (tableName.equals(TableName.valueOf("hbase:rsgroup"))){ - description = "The hbase:rsgroup table holds information about regionserver groups."; - } else if (tableName.equals(TableName.valueOf("hbase:replication"))) { - description = "The hbase:replication table tracks cross cluster replication through " + - "WAL file offsets."; - } - - - - -
Table NameFrag.Description
<% tableName %><% frags.get(tableName.getNameAsString()) != null ? frags.get(tableName.getNameAsString()) - .intValue() + "%" : "n/a" %><% description %>
- - - -<%def userTables> -<%java> - List tables = new ArrayList(); - String errorMessage = getUserTables(master, tables); - -<%if (tables.size() == 0 && errorMessage != null)%> -

<% errorMessage %>

- - -<%if (tables != null && tables.size() > 0)%> - - - - - - <%if (frags != null) %> - - - - - - - - - - - - - - - - - - <%for TableDescriptor desc : tables%> - <%java> - HTableDescriptor htDesc = new HTableDescriptor(desc); - TableName tableName = htDesc.getTableName(); - TableState tableState = master.getTableStateManager().getTableState(tableName); - Map> tableRegions = - master.getAssignmentManager().getRegionStates() - .getRegionByStateOfTable(tableName); - int openRegionsCount = tableRegions.get(RegionState.State.OPEN).size(); - int openingRegionsCount = tableRegions.get(RegionState.State.OPENING).size(); - int closedRegionsCount = tableRegions.get(RegionState.State.CLOSED).size(); - int closingRegionsCount = tableRegions.get(RegionState.State.CLOSING).size(); - int offlineRegionsCount = tableRegions.get(RegionState.State.OFFLINE).size(); - int splitRegionsCount = tableRegions.get(RegionState.State.SPLIT).size(); - int otherRegionsCount = 0; - for (List list: tableRegions.values()) { - otherRegionsCount += list.size(); - } - // now subtract known states - otherRegionsCount = otherRegionsCount - openRegionsCount - - offlineRegionsCount - splitRegionsCount - - openingRegionsCount - closedRegionsCount - - closingRegionsCount; - String encodedTableName = URLEncoder.encode(tableName.getNameAsString()); - - - - <%if (tableState.isDisabledOrDisabling()) %> <%else> - <%if (frags != null) %> - - - <%if (tableState.isDisabledOrDisabling()) %> <%else> - - <%if (openingRegionsCount > 0) %> <%else> - <%if (closedRegionsCount > 0) %> <%else> - <%if (closingRegionsCount > 0) %> <%else> - <%if (offlineRegionsCount > 0) %> <%else> - <%if (splitRegionsCount > 0) %> <%else> - - - - -

<% tables.size() %> table(s) in set. [Details]. Click count below to - see list of regions currently in 'state' designated by the column title. For 'Other' Region state, - browse to hbase:meta and adjust filter on 'Meta Entries' to - query on states other than those listed here. Queries may take a while if the hbase:meta table - is large.

- -
NamespaceNameFrag.StateRegionsDescription
OPENOPENINGCLOSEDCLOSINGOFFLINESPLITOther
<% tableName.getNamespaceAsString() %>><% URLEncoder.encode(tableName.getQualifierAsString()) %>><% URLEncoder.encode(tableName.getQualifierAsString()) %> <% frags.get(tableName.getNameAsString()) != null ? frags.get(tableName.getNameAsString()).intValue() + "%" : "n/a" %><% tableState.getState().name() %><% tableState.getState() %> <% openRegionsCount %><% openingRegionsCount %><% openingRegionsCount %> <% closedRegionsCount %><% closedRegionsCount %> <% closingRegionsCount %><% closingRegionsCount %> <% offlineRegionsCount %><% offlineRegionsCount %> <% splitRegionsCount %><% splitRegionsCount %> <% otherRegionsCount %><% htDesc.toStringCustomizedValues() %>
- - - - - -<%def deadRegionServers> - -<%if (deadServers != null && deadServers.size() > 0)%> -

Dead Region Servers

- - - - - - <%if !master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null - && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null - && master.getServerManager().getOnlineServersList().size() > 0 %> - - - - <%java> - List groups = null; - DeadServer deadServerUtil = master.getServerManager().getDeadServers(); - if (!master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null - && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null - && master.getServerManager().getOnlineServersList().size() > 0) { - groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection()); - } - ServerName [] deadServerNames = deadServers.toArray(new ServerName[deadServers.size()]); - Arrays.sort(deadServerNames); - for (ServerName deadServerName: deadServerNames) { - String rsGroupName = null; - if (groups != null){ - for (RSGroupInfo rsGroupInfo : groups) { - if (rsGroupInfo.containsServer(deadServerName.getAddress())) { - rsGroupName = rsGroupInfo.getName(); - break; - } - } - if (rsGroupName == null) { - rsGroupName = RSGroupInfo.DEFAULT_GROUP; - } - } - - - - - - <%if rsGroupName != null %> - - - - <%java> - } - - - - - - -
ServerNameStop timeRSGroup
<% deadServerName %><% deadServerUtil.getTimeOfDeath(deadServerName) %><% rsGroupName %>
Total: servers: <% deadServers.size() %>
- - - -<%def peerConfigs> -<%java> - List peers = null; - if (master.getReplicationPeerManager() != null) { - peers = master.getReplicationPeerManager().listPeers(null); - } - - - - - - - - - - - - - - - -<%if (peers != null && peers.size() > 0)%> - <%for ReplicationPeerDescription peer : peers %> - <%java> - String peerId = peer.getPeerId(); - ReplicationPeerConfig peerConfig = peer.getPeerConfig(); - - - - - - - - - - - - - - - - - -
Peer IdCluster KeyEndpointStateIsSerialBandwidthReplicateAllNamespacesExclude NamespacesTable CfsExclude Table Cfs
<% peerId %><% peerConfig.getClusterKey() %><% peerConfig.getReplicationEndpointImpl() %><% peer.isEnabled() ? "ENABLED" : "DISABLED" %><% peerConfig.isSerial() %><% peerConfig.getBandwidth() == 0? "UNLIMITED" : Strings.humanReadableInt(peerConfig.getBandwidth()) %><% peerConfig.replicateAllUserTables() %> - <% peerConfig.getNamespaces() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getNamespaces()).replaceAll(";", "; ") %> - - <% peerConfig.getExcludeNamespaces() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getExcludeNamespaces()).replaceAll(";", "; ") %> - - <% peerConfig.getTableCFsMap() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getTableCFsMap()).replaceAll(";", "; ") %> - - <% peerConfig.getExcludeTableCFsMap() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getExcludeTableCFsMap()).replaceAll(";", "; ") %> -
Total: <% (peers != null) ? peers.size() : 0 %>
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon deleted file mode 100644 index a00576786dbf..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon +++ /dev/null @@ -1,393 +0,0 @@ -<%doc> -Copyright The Apache Software Foundation - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -<%args> -HMaster master; -ServerManager serverManager; - - -<%import> - java.util.Collections; - java.util.List; - java.util.Map; - java.util.Set; - java.util.stream.Collectors; - org.apache.hadoop.hbase.master.HMaster; - org.apache.hadoop.hbase.RegionMetrics; - org.apache.hadoop.hbase.ServerMetrics; - org.apache.hadoop.hbase.Size; - org.apache.hadoop.hbase.RSGroupTableAccessor; - org.apache.hadoop.hbase.master.ServerManager; - org.apache.hadoop.hbase.net.Address; - org.apache.hadoop.hbase.rsgroup.RSGroupInfo; - org.apache.hadoop.util.StringUtils; - org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; - -<%java> -List groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection()); - -<%if (groups != null && groups.size() > 0)%> - -<%java> -RSGroupInfo [] rsGroupInfos = groups.toArray(new RSGroupInfo[groups.size()]); -Map collectServers = Collections.emptyMap(); -if (master.getServerManager() != null) { - collectServers = - master.getServerManager().getOnlineServers().entrySet().stream() - .collect(Collectors.toMap(p -> p.getKey().getAddress(), Map.Entry::getValue)); -} - - -
- -
-
- <& rsgroup_baseStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
- <& rsgroup_memoryStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
- <& rsgroup_requestStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
- <& rsgroup_storeStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
- <& rsgroup_compactStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
-
- - - -<%def rsgroup_baseStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - - - - -<%java> - int totalOnlineServers = 0; - int totalDeadServers = 0; - int totalTables = 0; - int totalRequests = 0; - int totalRegions = 0; - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String rsGroupName = rsGroupInfo.getName(); - int onlineServers = 0; - int deadServers = 0; - int tables = 0; - long requestsPerSecond = 0; - int numRegionsOnline = 0; - Set
servers = rsGroupInfo.getServers(); - for (Address server : servers) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - requestsPerSecond += sl.getRequestCountPerSecond(); - numRegionsOnline += sl.getRegionMetrics().size(); - //rsgroup total - totalRegions += sl.getRegionMetrics().size(); - totalRequests += sl.getRequestCountPerSecond(); - totalOnlineServers++; - onlineServers++; - } else { - totalDeadServers++; - deadServers++; - } - } - tables = rsGroupInfo.getTables().size(); - totalTables += tables; - double avgLoad = onlineServers == 0 ? 0 : - (double)numRegionsOnline / (double)onlineServers; - -
- - - - - - - - -<%java> -} - - - - - - - - - -
RSGroup NameNum. Online ServersNum. Dead ServersNum. TablesRequests Per SecondNum. RegionsAverage Load
<& rsGroupLink; rsGroupName=rsGroupName; &><% onlineServers %><% deadServers %><% tables %><% requestsPerSecond %><% numRegionsOnline %><% StringUtils.limitDecimalTo2(avgLoad) %>
Total:<% rsGroupInfos.length %><% totalOnlineServers %><% totalDeadServers %><% totalTables %><% totalRequests %><% totalRegions %><% StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %>
- - -<%def rsgroup_memoryStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - - -<%java> - final String ZEROMB = "0 MB"; - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String usedHeapStr = ZEROMB; - String maxHeapStr = ZEROMB; - String memstoreSizeStr = ZEROMB; - String rsGroupName = rsGroupInfo.getName(); - long usedHeap = 0; - long maxHeap = 0; - long memstoreSize = 0; - for (Address server : rsGroupInfo.getServers()) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - usedHeap += (long) sl.getUsedHeapSize().get(Size.Unit.MEGABYTE); - maxHeap += (long) sl.getMaxHeapSize().get(Size.Unit.MEGABYTE); - memstoreSize += (long) sl.getRegionMetrics().values().stream().mapToDouble( - rm -> rm.getMemStoreSize().get(Size.Unit.MEGABYTE)).sum(); - } - } - - if (usedHeap > 0) { - usedHeapStr = TraditionalBinaryPrefix.long2String(usedHeap - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (maxHeap > 0) { - maxHeapStr = TraditionalBinaryPrefix.long2String(maxHeap - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (memstoreSize > 0) { - memstoreSizeStr = TraditionalBinaryPrefix.long2String(memstoreSize - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - - - - - - - - -<%java> -} - -
RSGroup NameUsed HeapMax HeapMemstore Size
<& rsGroupLink; rsGroupName=rsGroupName; &><% usedHeapStr %><% maxHeapStr %><% memstoreSizeStr %>
- - -<%def rsgroup_requestStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - -<%java> - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String rsGroupName = rsGroupInfo.getName(); - long requestsPerSecond = 0; - long readRequests = 0; - long writeRequests = 0; - for (Address server : rsGroupInfo.getServers()) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - for (RegionMetrics rm : sl.getRegionMetrics().values()) { - readRequests += rm.getReadRequestCount(); - writeRequests += rm.getWriteRequestCount(); - } - requestsPerSecond += sl.getRequestCountPerSecond(); - } - } - - - - - - - -<%java> -} - -
RSGroup NameRequest Per SecondRead Request CountWrite Request Count
<& rsGroupLink; rsGroupName=rsGroupName; &><% requestsPerSecond %><% readRequests %><% writeRequests %>
- - - -<%def rsgroup_storeStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - - - - -<%java> - final String ZEROKB = "0 KB"; - final String ZEROMB = "0 MB"; - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String uncompressedStorefileSizeStr = ZEROMB; - String storefileSizeStr = ZEROMB; - String indexSizeStr = ZEROKB; - String bloomSizeStr = ZEROKB; - String rsGroupName = rsGroupInfo.getName(); - int numStores = 0; - long numStorefiles = 0; - long uncompressedStorefileSize = 0; - long storefileSize = 0; - long indexSize = 0; - long bloomSize = 0; - int count = 0; - for (Address server : rsGroupInfo.getServers()) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - for (RegionMetrics rm : sl.getRegionMetrics().values()) { - numStores += rm.getStoreCount(); - numStorefiles += rm.getStoreFileCount(); - uncompressedStorefileSize += rm.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); - storefileSize += rm.getStoreFileSize().get(Size.Unit.MEGABYTE); - indexSize += rm.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE); - bloomSize += rm.getBloomFilterSize().get(Size.Unit.KILOBYTE); - } - count++; - } - } - if (uncompressedStorefileSize > 0) { - uncompressedStorefileSizeStr = TraditionalBinaryPrefix. - long2String(uncompressedStorefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (storefileSize > 0) { - storefileSizeStr = TraditionalBinaryPrefix. - long2String(storefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (indexSize > 0) { - indexSizeStr = TraditionalBinaryPrefix. - long2String(indexSize * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - if (bloomSize > 0) { - bloomSizeStr = TraditionalBinaryPrefix. - long2String(bloomSize * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - - - - - - - - - - -<%java> -} - -
RSGroup NameNum. StoresNum. StorefilesStorefile Size UncompressedStorefile SizeIndex SizeBloom Size
<& rsGroupLink; rsGroupName=rsGroupName; &><% numStores %><% numStorefiles %><% uncompressedStorefileSizeStr %><% storefileSizeStr %><% indexSizeStr %><% bloomSizeStr %>
- - -<%def rsgroup_compactStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - - -<%java> - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String rsGroupName = rsGroupInfo.getName(); - int numStores = 0; - long totalCompactingCells = 0; - long totalCompactedCells = 0; - long remainingCells = 0; - long compactionProgress = 0; - for (Address server : rsGroupInfo.getServers()) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - for (RegionMetrics rl : sl.getRegionMetrics().values()) { - totalCompactingCells += rl.getCompactingCellCount(); - totalCompactedCells += rl.getCompactedCellCount(); - } - } - } - remainingCells = totalCompactingCells - totalCompactedCells; - String percentDone = ""; - if (totalCompactingCells > 0) { - percentDone = String.format("%.2f", 100 * - ((float) totalCompactedCells / totalCompactingCells)) + "%"; - } - - - - - - - - -<%java> -} - -
RSGroup NameNum. Compacting CellsNum. Compacted CellsRemaining CellsCompaction Progress
<& rsGroupLink; rsGroupName=rsGroupName; &><% totalCompactingCells %><% totalCompactedCells %><% remainingCells %><% percentDone %>
- - - -<%def rsGroupLink> - <%args> - String rsGroupName; - - ><% rsGroupName %> - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon deleted file mode 100644 index 2f1da65bf690..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon +++ /dev/null @@ -1,545 +0,0 @@ -<%doc> -Copyright The Apache Software Foundation - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -<%args> -List servers = null; -HMaster master; - - -<%import> - java.util.*; - org.apache.hadoop.hbase.master.HMaster; - org.apache.hadoop.hbase.procedure2.util.StringUtils; - org.apache.hadoop.hbase.replication.ReplicationLoadSource; - org.apache.hadoop.hbase.RegionMetrics; - org.apache.hadoop.hbase.RSGroupTableAccessor; - org.apache.hadoop.hbase.ServerMetrics; - org.apache.hadoop.hbase.ServerName; - org.apache.hadoop.hbase.Size; - org.apache.hadoop.hbase.net.Address; - org.apache.hadoop.hbase.rsgroup.RSGroupInfo; - org.apache.hadoop.hbase.util.VersionInfo; - org.apache.hadoop.hbase.util.Pair; - org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; - - -<%if (servers != null && servers.size() > 0)%> - -<%java> -ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]); -Arrays.sort(serverNames); - - -
- -
-
- <& baseStats; serverNames = serverNames; &> -
-
- <& memoryStats; serverNames = serverNames; &> -
-
- <& requestStats; serverNames = serverNames; &> -
-
- <& storeStats; serverNames = serverNames; &> -
-
- <& compactionStats; serverNames = serverNames; &> -
-
- <& replicationStats; serverNames = serverNames; &> -
-
-
- - - -<%def baseStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - <%if !master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null %> - <%if master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null && - master.getServerManager().getOnlineServersList().size() > 0 %> - - - - - - -<%java> - int totalRegions = 0; - int totalRequestsPerSecond = 0; - int inconsistentNodeNum = 0; - String state = "Normal"; - String masterVersion = VersionInfo.getVersion(); - Set decommissionedServers = new HashSet<>(master.listDecommissionedRegionServers()); - - String rsGroupName = "default"; - List groups; - Map server2GroupMap = new HashMap<>(); - if (!master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null - && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null - && master.getServerManager().getOnlineServersList().size() > 0) { - groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection()); - groups.forEach(group -> { - group.getServers().forEach(address -> server2GroupMap.put(address, group)); - }); - } - - for (ServerName serverName: serverNames) { - if (decommissionedServers.contains(serverName)) { - state = "Decommissioned"; - } - ServerMetrics sl = master.getServerManager().getLoad(serverName); - String version = master.getRegionServerVersion(serverName); - if (!masterVersion.equals(version)) { - inconsistentNodeNum ++; - } - - double requestsPerSecond = 0.0; - int numRegionsOnline = 0; - long lastContact = 0; - - if (sl != null) { - requestsPerSecond = sl.getRequestCountPerSecond(); - numRegionsOnline = sl.getRegionMetrics().size(); - totalRegions += sl.getRegionMetrics().size(); - totalRequestsPerSecond += sl.getRequestCountPerSecond(); - lastContact = (System.currentTimeMillis() - sl.getReportTimestamp())/1000; - } - long startcode = serverName.getStartcode(); - - if (!master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null - && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null - && master.getServerManager().getOnlineServersList().size() > 0) { - rsGroupName = server2GroupMap.get(serverName.getAddress()).getName(); - } - - - - - - - - - - <%if !master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null %> - <%if master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null && - master.getServerManager().getOnlineServersList().size() > 0 %> - - - - -<%java> -} - - - - - - -<%if inconsistentNodeNum > 0%> - -<%else> - - - - - -
ServerNameStateStart timeLast contactVersionRequests Per SecondNum. RegionsRSGroup
<& serverNameLink; serverName=serverName; &><% state %><% new Date(startcode) %><% TraditionalBinaryPrefix.long2String(lastContact, "s", 1) %><% version %><% String.format("%,.0f", requestsPerSecond) %><% String.format("%,d", numRegionsOnline) %><% rsGroupName %>
Total:<% servers.size() %><% inconsistentNodeNum %> nodes with inconsistent version<% totalRequestsPerSecond %><% totalRegions %>
- - -<%def memoryStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - -<%java> -final String ZEROMB = "0 MB"; -for (ServerName serverName: serverNames) { - String usedHeapStr = ZEROMB; - String maxHeapStr = ZEROMB; - String memStoreSizeMBStr = ZEROMB; - ServerMetrics sl = master.getServerManager().getLoad(serverName); - if (sl != null) { - long memStoreSizeMB = 0; - for (RegionMetrics rl : sl.getRegionMetrics().values()) { - memStoreSizeMB += rl.getMemStoreSize().get(Size.Unit.MEGABYTE); - } - if (memStoreSizeMB > 0) { - memStoreSizeMBStr = TraditionalBinaryPrefix.long2String(memStoreSizeMB - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - - double usedHeapSizeMB = sl.getUsedHeapSize().get(Size.Unit.MEGABYTE); - if (usedHeapSizeMB > 0) { - usedHeapStr = TraditionalBinaryPrefix.long2String((long) usedHeapSizeMB - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - double maxHeapSizeMB = sl.getMaxHeapSize().get(Size.Unit.MEGABYTE); - if (maxHeapSizeMB > 0) { - maxHeapStr = TraditionalBinaryPrefix.long2String((long) maxHeapSizeMB - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - - - - - - - -<%java> - } else { - -<& emptyStat; serverName=serverName; &> -<%java> - } -} - - -
ServerNameUsed HeapMax HeapMemstore Size
<& serverNameLink; serverName=serverName; &><% usedHeapStr %><% maxHeapStr %><% memStoreSizeMBStr %>
- - - -<%def requestStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - -<%java> -for (ServerName serverName: serverNames) { - -ServerMetrics sl = master.getServerManager().getLoad(serverName); -if (sl != null) { - long readRequestCount = 0; - long writeRequestCount = 0; - long filteredReadRequestCount = 0; - for (RegionMetrics rl : sl.getRegionMetrics().values()) { - readRequestCount += rl.getReadRequestCount(); - writeRequestCount += rl.getWriteRequestCount(); - filteredReadRequestCount += rl.getFilteredReadRequestCount(); - } - - - - - - - - -<%java> - } else { - -<& emptyStat; serverName=serverName; &> -<%java> - } -} - - -
ServerNameRequest Per SecondRead Request CountFiltered Read Request CountWrite Request Count
<& serverNameLink; serverName=serverName; &><% String.format("%,d", sl.getRequestCountPerSecond()) %><% String.format("%,d", readRequestCount) %><% String.format("%,d", filteredReadRequestCount) %><% String.format("%,d", writeRequestCount) %>
- - - -<%def storeStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - - - -<%java> -final String ZEROKB = "0 KB"; -final String ZEROMB = "0 MB"; -for (ServerName serverName: serverNames) { - - String storeUncompressedSizeMBStr = ZEROMB; - String storeFileSizeMBStr = ZEROMB; - String totalStaticIndexSizeKBStr = ZEROKB; - String totalStaticBloomSizeKBStr = ZEROKB; - ServerMetrics sl = master.getServerManager().getLoad(serverName); - if (sl != null) { - long storeCount = 0; - long storeFileCount = 0; - long storeUncompressedSizeMB = 0; - long storeFileSizeMB = 0; - long totalStaticIndexSizeKB = 0; - long totalStaticBloomSizeKB = 0; - for (RegionMetrics rl : sl.getRegionMetrics().values()) { - storeCount += rl.getStoreCount(); - storeFileCount += rl.getStoreFileCount(); - storeUncompressedSizeMB += rl.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); - storeFileSizeMB += rl.getStoreFileSize().get(Size.Unit.MEGABYTE); - totalStaticIndexSizeKB += rl.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE); - totalStaticBloomSizeKB += rl.getBloomFilterSize().get(Size.Unit.KILOBYTE); - } - if (storeUncompressedSizeMB > 0) { - storeUncompressedSizeMBStr = TraditionalBinaryPrefix. - long2String(storeUncompressedSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (storeFileSizeMB > 0) { - storeFileSizeMBStr = TraditionalBinaryPrefix. - long2String(storeFileSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (totalStaticIndexSizeKB > 0) { - totalStaticIndexSizeKBStr = TraditionalBinaryPrefix. - long2String(totalStaticIndexSizeKB * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - if (totalStaticBloomSizeKB > 0) { - totalStaticBloomSizeKBStr = TraditionalBinaryPrefix. - long2String(totalStaticBloomSizeKB * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - - - - - - - - - - -<%java> - } else { - -<& emptyStat; serverName=serverName; &> -<%java> - } -} - - -
ServerNameNum. StoresNum. StorefilesStorefile Size UncompressedStorefile SizeIndex SizeBloom Size
<& serverNameLink; serverName=serverName; &><% String.format("%,d", storeCount) %><% String.format("%,d", storeFileCount) %><% storeUncompressedSizeMBStr %><% storeFileSizeMBStr %><% totalStaticIndexSizeKBStr %><% totalStaticBloomSizeKBStr %>
- - -<%def compactionStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - -<%java> -for (ServerName serverName: serverNames) { - -ServerMetrics sl = master.getServerManager().getLoad(serverName); -if (sl != null) { -long totalCompactingCells = 0; -long totalCompactedCells = 0; -for (RegionMetrics rl : sl.getRegionMetrics().values()) { - totalCompactingCells += rl.getCompactingCellCount(); - totalCompactedCells += rl.getCompactedCellCount(); -} -String percentDone = ""; -if (totalCompactingCells > 0) { - percentDone = String.format("%.2f", 100 * - ((float) totalCompactedCells / totalCompactingCells)) + "%"; -} - - - - - - - - -<%java> - } else { - -<& emptyStat; serverName=serverName; &> -<%java> - } -} - - -
ServerNameNum. Compacting CellsNum. Compacted CellsRemaining CellsCompaction Progress
<& serverNameLink; serverName=serverName; &><% String.format("%,d", totalCompactingCells) %><% String.format("%,d", totalCompactedCells) %><% String.format("%,d", totalCompactingCells - totalCompactedCells) %><% percentDone %>
- - -<%def replicationStats> -<%args> - ServerName [] serverNames; - -<%java> - HashMap>> replicationLoadSourceMap - = master.getReplicationLoad(serverNames); - List peers = null; - if (replicationLoadSourceMap != null && replicationLoadSourceMap.size() > 0){ - peers = new ArrayList<>(replicationLoadSourceMap.keySet()); - Collections.sort(peers); - } - - -<%if (replicationLoadSourceMap != null && replicationLoadSourceMap.size() > 0) %> - -
-
    - <%java> - String active = "active"; - for (String peer : peers){ - -
  • >Peer <% peer %>
    • - <%java> - active = ""; - } - -
-
- <%java> - active = "active"; - for (String peer : peers){ - -
- - - - - - - - - <%for Pair pair: replicationLoadSourceMap.get(peer) %> - - - - - - - -
ServerAgeOfLastShippedOpSizeOfLogQueueReplicationLag
<& serverNameLink; serverName=pair.getFirst(); &><% StringUtils.humanTimeDiff(pair.getSecond().getAgeOfLastShippedOp()) %><% pair.getSecond().getSizeOfLogQueue() %><% pair.getSecond().getReplicationLag() == Long.MAX_VALUE ? "UNKNOWN" : StringUtils.humanTimeDiff(pair.getSecond().getReplicationLag()) %>
-
- <%java> - active = ""; - } - -
-

If the replication delay is UNKNOWN, that means this walGroup doesn't start replicate yet and it may get disabled.

-
-<%else> -

No Peers Metrics

- - - - - -<%def serverNameLink> - <%args> - ServerName serverName; - - <%java> - int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; - - - <%if infoPort > 0%> - <% serverName.getServerName() %> - <%else> - <% serverName.getServerName() %> - - - -<%def emptyStat> - <%args> - ServerName serverName; - - - <& serverNameLink; serverName=serverName; &> - - - - - - - - - - - - - - - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionVisualizerTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionVisualizerTmpl.jamon deleted file mode 100644 index 9a98cfefed7f..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionVisualizerTmpl.jamon +++ /dev/null @@ -1,119 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - - - - - -
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon deleted file mode 100644 index 82609aad7190..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon +++ /dev/null @@ -1,564 +0,0 @@ -<%doc> -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Template for rendering Block Cache tabs in RegionServer Status page. - -<%args> -CacheConfig cacheConfig; -Configuration config; -BlockCache bc; - -<%java> - String bcUrl = bc == null ? null : "http://hbase.apache.org/devapidocs/" + bc.getClass().getName().replaceAll("\\.", "/") + ".html"; - String bcName = bc == null ? null : bc.getClass().getSimpleName(); - BlockCache [] bcs = bc == null ? null : bc.getBlockCaches(); - boolean evictions = bcs != null && bcs.length > 1; - -<%import> -java.util.Map; -org.apache.hadoop.hbase.io.hfile.BlockCacheUtil; -org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile; -org.apache.hadoop.hbase.io.hfile.AgeSnapshot; -org.apache.hadoop.hbase.io.hfile.CachedBlock; -org.apache.hadoop.conf.Configuration; -org.apache.hadoop.hbase.io.hfile.CacheConfig; -org.apache.hadoop.hbase.io.hfile.BlockCache; -org.apache.hadoop.hbase.io.hfile.LruBlockCache; -org.apache.hadoop.hbase.io.hfile.bucket.BucketCacheStats; -org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator; -org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.Bucket; -org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; - -
- -
-
- <& bc_baseInfo; bc = bc; bcUrl = bcUrl; bcName = bcName; &> -
-
- <& bc_config; cacheConfig = cacheConfig &> -
-
- <& bc_stats; bc = bc &> -
-
- <& bc_l; bc = bcs == null? bc: bcs[0]; name = "L1"; evictions = evictions; &> -
-
- <& bc_l; bc = bcs == null? null: bcs.length <= 1? null: bcs[1]; name = "L2"; evictions = evictions; &> -
-
-
- -<%def bc_baseInfo> -<%args> - BlockCache bc; - String bcUrl; - String bcName; - -<%java> - BlockCache [] bcs = bc == null? null: bc.getBlockCaches(); - String bcl1Url = null; - String bcl1Name = null; - String bcl2Url = null; - String bcl2Name = null; - if (bcs != null) { - BlockCache bcl1 = bcs[0]; - if (bcl1 != null) { - bcl1Url = "http://hbase.apache.org/devapidocs/" + bcl1.getClass().getName().replaceAll("\\.", "/") + ".html"; - bcl1Name = bcl1.getClass().getSimpleName(); - } - if (bcs.length == 2) { - BlockCache bcl2 = bcs[1]; - bcl2Url = "http://hbase.apache.org/devapidocs/" + bcl2.getClass().getName().replaceAll("\\.", "/") + ".html"; - bcl2Name = bcl2.getClass().getSimpleName(); - } - } - - - - - - - - - - - - - -
AttributeValueDescription
Implementation<% bcName %>Block cache implementing class
-

See block cache in the HBase Reference Guide for help.

- - -<%def bc_config> -<%args> - CacheConfig cacheConfig; - -<%if cacheConfig == null %> -

CacheConfig is null

-<%else> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
AttributeValueDescription
Cache DATA on Read<% cacheConfig.shouldCacheDataOnRead() %>True if DATA blocks are cached on read - (INDEX & BLOOM blocks are always cached)
Cache DATA on Write<% cacheConfig.shouldCacheDataOnWrite() %>True if DATA blocks are cached on write.
Cache INDEX on Write<% cacheConfig.shouldCacheIndexesOnWrite() %>True if INDEX blocks are cached on write
Cache BLOOM on Write<% cacheConfig.shouldCacheBloomsOnWrite() %>True if BLOOM blocks are cached on write
Evict blocks on Close<% cacheConfig.shouldEvictOnClose() %>True if blocks are evicted from cache when an HFile - reader is closed
Cache DATA in compressed format<% cacheConfig.shouldCacheDataCompressed() %>True if DATA blocks are cached in their compressed form
Prefetch on Open<% cacheConfig.shouldPrefetchOnOpen() %>True if blocks are prefetched into cache on open
- - - -<%def evictions_tmpl> -<%args> - BlockCache bc; - -<%java> - AgeSnapshot ageAtEvictionSnapshot = bc.getStats().getAgeAtEvictionSnapshot(); - // Only show if non-zero mean and stddev as is the case in combinedblockcache - - - Evicted - <% String.format("%,d", bc.getStats().getEvictedCount()) %> - The total number of blocks evicted - - - Evictions - <% String.format("%,d", bc.getStats().getEvictionCount()) %> - The total number of times an eviction has occurred - - - Mean - <% String.format("%,d", (long)ageAtEvictionSnapshot.getMean()) %> - Mean age of Blocks at eviction time (seconds) - - - -<%def hits_tmpl> -<%args> - BlockCache bc; - -<%java> - int hitPeriods = 0; - for(int i=0; i -<%if hitPeriods > 0 %> - - - - Hits - <% String.format("%,d", bc.getStats().getHitCount()) %> - Number requests that were cache hits - - - Hits Caching - <% String.format("%,d", bc.getStats().getHitCachingCount()) %> - Cache hit block requests but only requests set to cache block if a miss - - - Misses - <% String.format("%,d", bc.getStats().getMissCount()) %> - Block requests that were cache misses but set to cache missed blocks - - - Misses Caching - <% String.format("%,d", bc.getStats().getMissCachingCount()) %> - Block requests that were cache misses but only requests set to use block cache - - - All Time Hit Ratio - <% String.format("%,.2f", bc.getStats().getHitRatio() * 100) %><% "%" %> - Hit Count divided by total requests count - - <%for int i=0; i - - Hit Ratio for period starting at <% bc.getStats().getWindowPeriods()[i] %> - <%if bc.getStats().getRequestCounts()[i] > 0 %> - <% String.format("%,.2f", ((double)bc.getStats().getHitCounts()[i] / (double)bc.getStats().getRequestCounts()[i]) * 100.0) %><% "%" %> - <%else> - No requests - - Hit Count divided by total requests count over the <% i %>th period of <% bc.getStats().getPeriodTimeInMinutes() %> minutes - - - <%if hitPeriods > 0 %> - - -
- - - - - - -
- - - - <%if bc.getStats().getPeriodTimeInMinutes() > 0 %> - - Last <% bc.getStats().getNumPeriodsInWindow()*bc.getStats().getPeriodTimeInMinutes() %> minutes Hit Ratio - <% String.format("%,.2f", bc.getStats().getHitRatioPastNPeriods() * 100.0) %><% "%" %> - Hit Count divided by total requests count for the last <% bc.getStats().getNumPeriodsInWindow()*bc.getStats().getPeriodTimeInMinutes() %> minutes - - - - - -<%def bc_stats> -<%args> - BlockCache bc; - -<%if bc == null %> -

BlockCache is null

-<%else> - - - - - - - - - - - - - - - - - - - - - - <& evictions_tmpl; bc = bc; &> - <& hits_tmpl; bc = bc; &> -
AttributeValueDescription
Size<% TraditionalBinaryPrefix.long2String(bc.getCurrentSize(), - "B", 1) %>Current size of block cache in use
Free<% TraditionalBinaryPrefix.long2String(bc.getFreeSize(), - "B", 1) %>The total free memory currently available to store more cache entries
Count<% String.format("%,d", bc.getBlockCount()) %>Number of blocks in block cache
-

If block cache is made up of more than one cache -- i.e. a L1 and a L2 -- then the above -are combined counts. Request count is sum of hits and misses.

- - - -<%def bc_l> -<%args> - BlockCache bc; - String name; - boolean evictions; - -<%if bc == null %> -

No <% name %> deployed

-<%else> -<& block_cache; bc = bc; name = name; evictions = evictions; &> - - - -<%def block_cache> -<%args> - BlockCache bc; - String name; - boolean evictions; - -<%java> - String bcUrl = "http://hbase.apache.org/devapidocs/" + bc.getClass().getName().replaceAll("\\.", "/") + ".html"; - String bcName = bc.getClass().getSimpleName(); - int maxCachedBlocksByFile = BlockCacheUtil.getMaxCachedBlocksByFile(config); - - boolean lru = bc instanceof LruBlockCache; - - boolean bucketCache = bc.getClass().getSimpleName().equals("BucketCache"); - BucketCacheStats bucketCacheStats = null; - BucketAllocator bucketAllocator = null; - - if (bucketCache) { - bucketCacheStats = (BucketCacheStats)bc.getStats(); - bucketAllocator = ((BucketCache)bc).getAllocator(); - } - - - - - - - - - - - - -<%if bucketCache %> - - - - - - - - - - - - - - - - -<%if !bucketCache %> - - - - - - -<%if lru %> - - - - - - - - - - - - - - - - -<%if !bucketCache %> - - - - - - -<%if lru %> - - - - - - - - - - - - <& evictions_tmpl; bc = bc; &> -<& hits_tmpl; bc = bc; &> - -<%if bucketCache %> - - - - - - - - - - - -
AttributeValueDescription
Implementation<% bc.getClass().getSimpleName() %>Class implementing this block cache Level
Implementation<% ((BucketCache)bc).getIoEngine() %>IOEngine
Cache Size Limit<% TraditionalBinaryPrefix.long2String(bc.getMaxSize(), "B", 1) %>Max size of cache
Block Count<% String.format("%,d", bc.getBlockCount()) %>Count of Blocks
Data Block Count<% String.format("%,d", bc.getDataBlockCount()) %>Count of DATA Blocks
Index Block Count<% String.format("%,d", ((LruBlockCache)bc).getIndexBlockCount()) %>Count of INDEX Blocks
Bloom Block Count<% String.format("%,d", ((LruBlockCache)bc).getBloomBlockCount()) %>Count of BLOOM Blocks
Size of Blocks<% TraditionalBinaryPrefix.long2String(bc.getCurrentSize(), "B", 1) %>Size of Blocks
Size of Data Blocks<% TraditionalBinaryPrefix.long2String(bc.getCurrentDataSize(), "B", 1) %>Size of DATA Blocks
Size of Index Blocks<% TraditionalBinaryPrefix.long2String(((LruBlockCache)bc).getCurrentIndexSize(), "B", 1) %>Size of INDEX Blocks
Size of Bloom Blocks<% TraditionalBinaryPrefix.long2String(((LruBlockCache)bc).getCurrentBloomSize(), "B", 1) %>Size of BLOOM Blocks
Hits per Second<% bucketCacheStats.getIOHitsPerSecond() %>Block gets against this cache per second
Time per Hit<% bucketCacheStats.getIOTimePerHit() %>Time per cache hit
- -<%doc>Call through to block cache Detail rendering template -

-View block cache as JSON | Block cache as JSON by file -<%if bc.getBlockCount() > maxCachedBlocksByFile %> -
-Note: JSON view of block cache will be incomplete, because block count <% bc.getBlockCount() %> is greater than hbase.ui.blockcache.by.file.max value of <% maxCachedBlocksByFile %>. -Increase that value to get a complete picture. - -

- - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon deleted file mode 100644 index ce55aaf12702..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon +++ /dev/null @@ -1,80 +0,0 @@ -<%doc> -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -This template is used to give views on an individual block cache as JSON. - -<%args> -CacheConfig cacheConfig; -Configuration conf; -String bcn; -String bcv; -BlockCache blockCache; - -<%import> -java.util.*; -org.apache.hadoop.conf.Configuration; -org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile; -org.apache.hadoop.hbase.io.hfile.BlockCacheUtil; -org.apache.hadoop.hbase.io.hfile.CachedBlock; -org.apache.hadoop.hbase.io.hfile.CacheConfig; -org.apache.hadoop.hbase.io.hfile.BlockCache; -org.apache.hadoop.hbase.io.hfile.bucket.BucketCacheStats; -org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator; -org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.Bucket; -org.apache.hadoop.util.StringUtils; - -<%java> - BlockCache bc = blockCache; - BlockCache [] bcs = bc == null ? null : bc.getBlockCaches(); - if (bcn.equals("L1")) { - bc = bcs == null || bcs.length == 0? bc: bcs[0]; - } else { - if (bcs == null || bcs.length < 2) { - System.out.println("There is no L2 block cache"); - return; - } - bc = bcs[1]; - } - if (bc == null) { - System.out.println("There is no block cache"); - return; - } - CachedBlocksByFile cbsbf = BlockCacheUtil.getLoadedCachedBlocksByFile(conf, bc); - -<%if bcv.equals("file") %><& bc_by_file; cbsbf = cbsbf; &><%else>[ <% BlockCacheUtil.toJSON(bc) %>, <% BlockCacheUtil.toJSON(cbsbf) %> ] -<%java> -cbsbf = null; - - -<%def bc_by_file> -<%args> - CachedBlocksByFile cbsbf; - -<%java> - boolean firstEntry = true; - -[<%for Map.Entry> e: cbsbf.getCachedBlockStatsByFile().entrySet() %> - <%if !firstEntry %>,<% BlockCacheUtil.toJSON(e.getKey(), e.getValue()) %> - <%java> - if (firstEntry) { - firstEntry = false; - } - -] - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon deleted file mode 100644 index 4b8047b9e446..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ /dev/null @@ -1,330 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%args> -HRegionServer regionServer; -String filter = "general"; -String format = "html"; -String bcn = ""; -String bcv = ""; - -<%import> -java.util.*; -org.apache.hadoop.hbase.regionserver.HRegionServer; -org.apache.hadoop.hbase.client.RegionInfo; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.HBaseConfiguration; -org.apache.hadoop.hbase.io.hfile.CacheConfig; -org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo; -org.apache.hadoop.hbase.util.JvmVersion; -org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; - -<%doc>If json AND bcn is NOT an empty string presume it a block cache view request. -<%if format.equals("json") && bcn != null && bcn.length() > 0 %> - <& BlockCacheViewTmpl; conf = regionServer.getConfiguration(); cacheConfig = new CacheConfig(regionServer.getConfiguration()); bcn = bcn; bcv = bcv; blockCache = regionServer.getBlockCache().orElse(null) &> - <%java return; %> -<%elseif format.equals("json") %> - <& ../common/TaskMonitorTmpl; filter = filter; format = "json" &> - <%java return; %> - -<%java> - ServerInfo serverInfo = ProtobufUtil.getServerInfo(null, regionServer.getRSRpcServices()); - ServerName serverName = ProtobufUtil.toServerName(serverInfo.getServerName()); - List onlineRegions = ProtobufUtil.getOnlineRegions(regionServer.getRSRpcServices()); - MasterAddressTracker masterAddressTracker = regionServer.getMasterAddressTracker(); - ServerName masterServerName = masterAddressTracker == null ? null - : masterAddressTracker.getMasterAddress(); - int infoPort = masterAddressTracker == null ? 0 : masterAddressTracker.getMasterInfoPort(); - - -<%class> - public String formatZKString() { - StringBuilder quorums = new StringBuilder(); - String zkQuorum = regionServer.getZooKeeper().getQuorum(); - - if (null == zkQuorum) { - return quorums.toString(); - } - - String[] zks = zkQuorum.split(","); - - if (zks.length == 0) { - return quorums.toString(); - } - - for(int i = 0; i < zks.length; ++i) { - quorums.append(zks[i].trim()); - - if (i != (zks.length - 1)) { - quorums.append("
"); - } - } - - return quorums.toString(); - } - - - - - - - - HBase Region Server: <% serverName.getHostname() %> - - - - - - - - - - - - -
-
-
-

RegionServer <% serverName %>

-
-
-
- -
-

Server Metrics

- <& ServerMetricsTmpl; mWrap = regionServer.getMetrics().getRegionServerWrapper(); - mServerWrap = regionServer.getRpcServer().getMetrics().getHBaseServerWrapper(); - bbAllocator = regionServer.getRpcServer().getByteBuffAllocator(); &> -
- -
-

Block Cache

- <& BlockCacheTmpl; cacheConfig = new CacheConfig(regionServer.getConfiguration()); config = regionServer.getConfiguration(); bc = regionServer.getBlockCache().orElse(null) &> -
- -
- <& ../common/TaskMonitorTmpl; filter = filter; parent = "/rs-status" &> -
- -
-

Regions

- <& RegionListTmpl; regionServer = regionServer; onlineRegions = onlineRegions; &> -
- -
-

Replication Status

- <& ReplicationStatusTmpl; regionServer = regionServer; &> -
- -
-

Software Attributes

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <%escape #n> - - - - - - - - - - - - - - - - - - -
Attribute NameValueDescription
JVM Version<% JvmVersion.getVersion() %>JVM vendor and version
HBase Version<% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision
HBase Compiled<% org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <% org.apache.hadoop.hbase.util.VersionInfo.getUser() %>When HBase version was compiled and by whom
HBase Source Checksum<% org.apache.hadoop.hbase.util.VersionInfo.getSrcChecksum() %>HBase source SHA512 checksum
Hadoop Version<% org.apache.hadoop.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.util.VersionInfo.getRevision() %>Hadoop version and revision
Hadoop Compiled<% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %>When Hadoop version was compiled and by whom
Hadoop Source Checksum<% org.apache.hadoop.util.VersionInfo.getSrcChecksum() %>Hadoop source MD5 checksum
ZooKeeper Client Version<% org.apache.zookeeper.Version.getVersion() %>, revision=<% org.apache.zookeeper.Version.getRevisionHash() %>ZooKeeper client version and revision hash
ZooKeeper Client Compiled<% org.apache.zookeeper.Version.getBuildDate() %>When ZooKeeper client version was compiled
ZooKeeper Quorum<% formatZKString() %>Addresses of all registered ZK servers
Coprocessors<% java.util.Arrays.toString(regionServer.getRegionServerCoprocessors()) %>Coprocessors currently loaded by this regionserver
RS Start Time<% new Date(regionServer.getStartcode()) %>Date stamp of when this region server was started
HBase Master - <%if masterServerName == null %> - No master found - <%else> - <%java> - String host = masterServerName.getHostname() + ":" + infoPort; - String url = "//" + host + "/master-status"; - - <% host %> - - Address of HBase Master
-
-
-
- - - - - - - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon deleted file mode 100644 index c4b947308bad..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon +++ /dev/null @@ -1,338 +0,0 @@ -<%doc> - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -<%args> - HRegionServer regionServer; - List onlineRegions; - -<%import> - java.util.*; - org.apache.commons.lang3.time.FastDateFormat; - org.apache.hadoop.hbase.regionserver.HRegionServer; - org.apache.hadoop.hbase.util.Bytes; - org.apache.hadoop.hbase.client.RegionInfo; - org.apache.hadoop.hbase.client.RegionInfoDisplay; - org.apache.hadoop.hbase.regionserver.Region; - org.apache.hadoop.hbase.regionserver.HStoreFile; - org.apache.hadoop.hbase.regionserver.Store; - org.apache.hadoop.hbase.regionserver.StoreFile; - org.apache.hadoop.hbase.ServerName; - org.apache.hadoop.hbase.HBaseConfiguration; - org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; - org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo; - org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionLoad; - org.apache.hadoop.hbase.client.RegionReplicaUtil; - org.apache.hadoop.hbase.regionserver.MetricsRegionWrapper; - org.apache.hadoop.util.StringUtils; - org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; - -<%if (onlineRegions != null && onlineRegions.size() > 0) %> - - <%java> - Collections.sort(onlineRegions, RegionInfo.COMPARATOR); - - -
- -
-
- <& baseInfo; onlineRegions = onlineRegions; &> -
-
- <& requestStats; onlineRegions = onlineRegions; &> -
-
- <& storeStats; onlineRegions = onlineRegions; &> -
-
- <& memstoreStats; onlineRegions = onlineRegions; &> -
-
- <& compactStats; onlineRegions = onlineRegions; &> -
-
-
-

Region names are made of the containing table's name, a comma, - the start key, a comma, and a randomly generated region id. To illustrate, - the region named - domains,apache.org,5464829424211263407 is party to the table - domains, has an id of 5464829424211263407 and the first key - in the region is apache.org. The hbase:meta 'table' is an internal - system table (or a 'catalog' table in db-speak). - The hbase:meta table keeps a list of all regions in the system. The empty key is used to denote - table start and table end. A region with an empty start key is the first region in a table. - If a region has both an empty start key and an empty end key, it's the only region in the - table. See HBase Home for further explication.

-<%else> -

Not serving regions

- - -<%def baseInfo> -<%args> - List onlineRegions; - - - - - - - - - - - - - <%for RegionInfo r: onlineRegions %> - - <%java> - String displayName = RegionInfoDisplay.getRegionNameAsStringForDisplay(r, - regionServer.getConfiguration()); - - - - - - - - -
Region NameStart KeyEnd KeyReplicaID
<% displayName %><% Bytes.toStringBinary(RegionInfoDisplay.getStartKeyForDisplay(r, - regionServer.getConfiguration())) %><% Bytes.toStringBinary(RegionInfoDisplay.getEndKeyForDisplay(r, - regionServer.getConfiguration())) %><% r.getReplicaId() %>
- - -<%def requestStats> -<%args> - List onlineRegions; - - - - - - - - - - - - - <%for RegionInfo r: onlineRegions %> - - - <%java> - RegionLoad load = regionServer.createRegionLoad(r.getEncodedName()); - String displayName = RegionInfoDisplay.getRegionNameAsStringForDisplay(r, - regionServer.getConfiguration()); - - - <%if load != null %> - - - - - - - -
Region NameRead Request CountFiltered Read Request CountWrite Request Count
<% displayName %><% String.format("%,1d", load.getReadRequestsCount()) %><% String.format("%,1d", load.getFilteredReadRequestsCount()) %><% String.format("%,1d", load.getWriteRequestsCount()) %>
- - - -<%def storeStats> -<%args> - List onlineRegions; - - - - - - - - - - - - - - - - - - - <%for RegionInfo r: onlineRegions %> - - - <%java> - final String ZEROMB = "0 MB"; - final String ZEROKB = "0 KB"; - String uncompressedStorefileSizeStr = ZEROMB; - String storefileSizeStr = ZEROMB; - String indexSizeStr = ZEROKB; - String bloomSizeStr = ZEROKB; - RegionLoad load = regionServer.createRegionLoad(r.getEncodedName()); - String displayName = RegionInfoDisplay.getRegionNameAsStringForDisplay(r, - regionServer.getConfiguration()); - if (load != null) { - long uncompressedStorefileSize = load.getStoreUncompressedSizeMB(); - long storefileSize = load.getStorefileSizeMB(); - long indexSize = load.getTotalStaticIndexSizeKB(); - long bloomSize = load.getTotalStaticBloomSizeKB(); - if (uncompressedStorefileSize > 0) { - uncompressedStorefileSizeStr = TraditionalBinaryPrefix.long2String( - uncompressedStorefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (storefileSize > 0) { - storefileSizeStr = TraditionalBinaryPrefix.long2String(storefileSize - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if(indexSize > 0) { - indexSizeStr = TraditionalBinaryPrefix.long2String(indexSize - * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - if (bloomSize > 0) { - bloomSizeStr = TraditionalBinaryPrefix.long2String(bloomSize - * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - } - long lenOfBiggestCellInRegion = -1L; - Region region = regionServer.getRegion(r.getEncodedName()); - if (region != null) { - List stores = region.getStores(); - for (Store store : stores) { - Collection storeFiles = store.getStorefiles(); - for (StoreFile sf : storeFiles) { - long lenOfBiggestCell = ((HStoreFile)sf).getFileInfo().getHFileInfo().getLenOfBiggestCell(); - if (lenOfBiggestCellInRegion < lenOfBiggestCell) { - lenOfBiggestCellInRegion = lenOfBiggestCell; - } - } - } - } - - - - <%if load != null %> - - - - - - - - - - - - - -
Region NameNum. StoresNum. StorefilesStorefile Size UncompressedStorefile SizeIndex SizeBloom SizeData LocalityLen Of Biggest Cell% Cached
<% displayName %><% String.format("%,1d", load.getStores()) %><% String.format("%,1d", load.getStorefiles()) %><% uncompressedStorefileSizeStr %><% storefileSizeStr %><% indexSizeStr %><% bloomSizeStr %><% load.getDataLocality() %><% String.format("%,1d", lenOfBiggestCellInRegion) %><% StringUtils.formatPercent(load.getCurrentRegionCachedRatio(), 2) %>
- - - -<%def compactStats> -<%args> - List onlineRegions; - - - - - - - - - - - - - - <%for RegionInfo r: onlineRegions %> - - - <%java> - RegionLoad load = regionServer.createRegionLoad(r.getEncodedName()); - String percentDone = ""; - String compactTime = ""; - if (load != null) { - if (load.getTotalCompactingKVs() > 0) { - percentDone = String.format("%.2f", 100 * - ((float) load.getCurrentCompactedKVs() / load.getTotalCompactingKVs())) + "%"; - } - if (load.getLastMajorCompactionTs() > 0) { - FastDateFormat fdf = FastDateFormat.getInstance("yyyy-MM-dd HH:mm (ZZ)"); - compactTime = fdf.format(load.getLastMajorCompactionTs()); - } - } - String displayName = RegionInfoDisplay.getRegionNameAsStringForDisplay(r, - regionServer.getConfiguration()); - - - <%if load != null %> - - - - - - - - -
Region NameNum. Compacting CellsNum. Compacted CellsCompaction ProgressLast Major Compaction
<% displayName %><% String.format("%,1d", load.getTotalCompactingKVs()) %><% String.format("%,1d", load.getCurrentCompactedKVs()) %><% percentDone %><% compactTime %>
- - -<%def memstoreStats> -<%args> - List onlineRegions; - - - - - - - - - - - <%for RegionInfo r: onlineRegions %> - - - <%java> - final String ZEROMB = "0 MB"; - String memStoreSizeMBStr = ZEROMB; - RegionLoad load = regionServer.createRegionLoad(r.getEncodedName()); - String displayName = RegionInfoDisplay.getRegionNameAsStringForDisplay(r, - regionServer.getConfiguration()); - if (load != null) { - long memStoreSizeMB = load.getMemStoreSizeMB(); - if (memStoreSizeMB > 0) { - memStoreSizeMBStr = TraditionalBinaryPrefix.long2String( - memStoreSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - } - - - <%if load != null %> - - - - - -
Region NameMemstore Size
<% displayName %><% memStoreSizeMBStr %>
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ReplicationStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ReplicationStatusTmpl.jamon deleted file mode 100644 index 273b26aecd36..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ReplicationStatusTmpl.jamon +++ /dev/null @@ -1,105 +0,0 @@ -<%doc> - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -<%args> - HRegionServer regionServer; - -<%import> - java.util.*; - java.util.Map.Entry; - org.apache.hadoop.hbase.procedure2.util.StringUtils; - org.apache.hadoop.hbase.regionserver.HRegionServer; - org.apache.hadoop.hbase.replication.regionserver.ReplicationStatus; - - -<%java> - Map walGroupsReplicationStatus = regionServer.getWalGroupsReplicationStatus(); - - -<%if (walGroupsReplicationStatus != null && walGroupsReplicationStatus.size() > 0) %> - -
- -
-
- <& currentLog; metrics = walGroupsReplicationStatus; &> -
-
- <& replicationDelay; metrics = walGroupsReplicationStatus; &> -
-
-
-

If the replication delay is UNKNOWN, that means this walGroup doesn't start replicate yet and it may get disabled. - If the size of log is 0, it means we are replicating current HLog, thus we can't get accurate size since it's not closed yet.

- -<%else> -

No Replication Metrics for Peers

- - -<%def currentLog> -<%args> - Map metrics; - - - - - - - - - - - <%for Map.Entry entry: metrics.entrySet() %> - - - - - - - - - -
PeerIdWalGroupCurrent LogSizeQueue SizeOffset
<% entry.getValue().getPeerId() %><% entry.getValue().getWalGroup() %><% entry.getValue().getCurrentPath() %> <% StringUtils.humanSize(entry.getValue().getFileSize()) %><% entry.getValue().getQueueSize() %><% StringUtils.humanSize(entry.getValue().getCurrentPosition()) %>
- - -<%def replicationDelay> -<%args> - Map metrics; - - - - - - - - - - <%for Map.Entry entry: metrics.entrySet() %> - - - - - - - - -
PeerIdWalGroupCurrent LogLast Shipped AgeReplication Delay
<% entry.getValue().getPeerId() %><% entry.getValue().getWalGroup() %><% entry.getValue().getCurrentPath() %> <% StringUtils.humanTimeDiff(entry.getValue().getAgeOfLastShippedOp()) %><% entry.getValue().getReplicationDelay() == Long.MAX_VALUE ? "UNKNOWN" : StringUtils.humanTimeDiff(entry.getValue().getReplicationDelay()) %>
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon deleted file mode 100644 index 84161a80381c..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon +++ /dev/null @@ -1,269 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%args> -MetricsRegionServerWrapper mWrap; -MetricsHBaseServerWrapper mServerWrap; -ByteBuffAllocator bbAllocator; - -<%import> -java.util.*; -org.apache.hadoop.hbase.regionserver.HRegionServer; -org.apache.hadoop.hbase.io.ByteBuffAllocator; -org.apache.hadoop.hbase.ipc.MetricsHBaseServerWrapper; -org.apache.hadoop.hbase.regionserver.MetricsRegionServerWrapper; -org.apache.hadoop.hbase.util.Bytes; -org.apache.hadoop.hbase.HRegionInfo; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.HBaseConfiguration; -org.apache.hadoop.hbase.protobuf.ProtobufUtil; -org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; -org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad; -org.apache.hadoop.hbase.util.DirectMemoryUtils; -org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; -java.lang.management.MemoryUsage; -org.apache.hadoop.hbase.io.util.MemorySizeUtil; - -
- -
-
- <& baseStats; mWrap = mWrap &> -
-
- <& memoryStats; mWrap = mWrap &> -
-
- <& requestStats; mWrap = mWrap &> -
-
- <& walStats; mWrap = mWrap &> -
-
- <& storeStats; mWrap = mWrap &> -
-
- <& queueStats; mWrap = mWrap; mServerWrap = mServerWrap; &> -
-
- <& byteBuffAllocatorStats; bbAllocator = bbAllocator; &> -
-
-
- -<%def baseStats> -<%args> - MetricsRegionServerWrapper mWrap; - - - - - - - - - - - - - - - - - -
Requests Per SecondNum. RegionsBlock localityBlock locality (Secondary replicas)Slow WAL Append Count
<% String.format("%.0f", mWrap.getRequestsPerSecond()) %><% mWrap.getNumOnlineRegions() %><% String.format("%.3f",mWrap.getPercentFileLocal()) %><% "%" %><% String.format("%.3f",mWrap.getPercentFileLocalSecondaryRegions()) %><% "%" %><% mWrap.getNumWALSlowAppend() %>
- - -<%def memoryStats> -<%args> -MetricsRegionServerWrapper mWrap; - -<%java - long usedHeap = -1L; - long maxHeap = -1L; - final MemoryUsage usage = MemorySizeUtil.safeGetHeapMemoryUsage(); - if (usage != null) { - maxHeap = usage.getMax(); - usedHeap = usage.getUsed(); - } -%> - - - - - - - - - - - - - - - - - - - - - -
Used HeapMax HeapDirect Memory UsedDirect Memory ConfiguredMemstore On-Heap Size / LimitMemstore Off-Heap Size / LimitMemstore Data Size (On&&Off Heap)
- <% TraditionalBinaryPrefix.long2String(usedHeap, "B", 1) %> - - <% TraditionalBinaryPrefix.long2String(maxHeap, "B", 1) %> - - <% TraditionalBinaryPrefix.long2String(DirectMemoryUtils.getDirectMemoryUsage(), "B", 1) %> - - <% TraditionalBinaryPrefix.long2String(DirectMemoryUtils.getDirectMemorySize(), "B", 1) %> - - <% TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreSize(), "B", 1) + " / " - + TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreLimit(), "B", 1) %> - - <% TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreSize(), "B", 1) + " / " - + TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreLimit(), "B", 1) %> - - <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B", 1) %> -
- - -<%def walStats> -<%args> -MetricsRegionServerWrapper mWrap; - - - - - - - - - - - - - - -
Num. WAL FilesSize. WAL FilesWAL exclude DNs
<% mWrap.getNumWALFiles() %><% TraditionalBinaryPrefix.long2String(mWrap.getWALFileSize(), "B", 1) %> - <%for String exclude: mWrap.getWALExcludeDNs() %> - <% exclude %>
- -
- - -<%def storeStats> -<%args> -MetricsRegionServerWrapper mWrap; - - - - - - - - - - - - - - - - - -
Num. StoresNum. StorefilesRoot Index SizeIndex SizeBloom Size
<% mWrap.getNumStores() %><% mWrap.getNumStoreFiles() %><% TraditionalBinaryPrefix.long2String(mWrap.getStoreFileIndexSize(), "B", 1) %><% TraditionalBinaryPrefix.long2String(mWrap.getTotalStaticIndexSize(), "B", 1) %><% TraditionalBinaryPrefix.long2String(mWrap.getTotalStaticBloomSize(), "B", 1) %>
- - - -<%def requestStats> -<%args> -MetricsRegionServerWrapper mWrap; - - - - - - - - - - - - - - -
Request Per SecondRead Request CountFiltered Read Request CountWrite Request Count
<% String.format("%.0f", mWrap.getRequestsPerSecond()) %><% mWrap.getReadRequestsCount() %><% mWrap.getFilteredReadRequestsCount() %><% mWrap.getWriteRequestsCount() %>
- - -<%def queueStats> -<%args> -MetricsRegionServerWrapper mWrap; -MetricsHBaseServerWrapper mServerWrap; - - - - - - - - - - - - - - - - - - - -
Compaction Queue LengthFlush Queue LengthPriority Call Queue LengthGeneral Call Queue LengthReplication Call Queue LengthTotal Call Queue Size
<% mWrap.getCompactionQueueSize() %><% mWrap.getFlushQueueSize() %><% mServerWrap.getPriorityQueueLength() %><% mServerWrap.getGeneralQueueLength() %><% mServerWrap.getReplicationQueueLength() %><% TraditionalBinaryPrefix.long2String(mServerWrap.getTotalQueueSize(), "B", 1) %>
- - -<%def byteBuffAllocatorStats> -<%args> -ByteBuffAllocator bbAllocator; - - - - - - - - - - - - - - - - - - -
Total Heap AllocationTotal Pool AllocationHeap Allocation RatioTotal Buffer CountUsed Buffer CountBuffer Size
<% TraditionalBinaryPrefix.long2String(ByteBuffAllocator.getHeapAllocationBytes(bbAllocator, ByteBuffAllocator.HEAP), "B", 1) %><% TraditionalBinaryPrefix.long2String(bbAllocator.getPoolAllocationBytes(), "B", 1) %><% String.format("%.3f", ByteBuffAllocator.getHeapAllocationRatio(bbAllocator, ByteBuffAllocator.HEAP) * 100) %><% "%" %><% bbAllocator.getTotalBufferCount() %><% bbAllocator.getUsedBufferCount() %><% TraditionalBinaryPrefix.long2String(bbAllocator.getBufferSize(), "B", 1) %>
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/tool/CanaryStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/tool/CanaryStatusTmpl.jamon deleted file mode 100644 index 031f3445518d..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/tool/CanaryStatusTmpl.jamon +++ /dev/null @@ -1,158 +0,0 @@ - -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%args> -RegionStdOutSink sink; - -<%import> -java.util.Map; -java.util.concurrent.atomic.LongAdder; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.tool.CanaryTool.RegionStdOutSink; -org.apache.hadoop.hbase.util.JvmVersion; - - - - - - - - Canary - - - - - - - - - - - -
-
-

Failed Servers

- <%java> - Map perServerFailuresCount = sink.getPerServerFailuresCount(); - - - - - - - <%if (perServerFailuresCount != null && perServerFailuresCount.size() > 0)%> - <%for Map.Entry entry : perServerFailuresCount.entrySet() %> - - - - - - - -
ServerFailures Count
<& serverNameLink ; serverName = entry.getKey() &><% entry.getValue() %>
Total Failed Servers: <% (perServerFailuresCount != null) ? perServerFailuresCount.size() : 0 %>
-
-
-

Failed Tables

- <%java> - Map perTableFailuresCount = sink.getPerTableFailuresCount(); - - - - - - - <%if (perTableFailuresCount != null && perTableFailuresCount.size() > 0)%> - <%for Map.Entry entry : perTableFailuresCount.entrySet()%> - - - - - - - -
TableFailures Count
<% entry.getKey() %><% entry.getValue() %>
Total Failed Tables: <% (perTableFailuresCount != null) ? perTableFailuresCount.size() : 0 %>
-
- -
-

Software Attributes

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Attribute NameValueDescription
JVM Version<% JvmVersion.getVersion() %>JVM vendor and version
HBase Version<% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision
HBase Compiled<% org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <% org.apache.hadoop.hbase.util.VersionInfo.getUser() %>When HBase version was compiled and by whom
Hadoop Version<% org.apache.hadoop.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.util.VersionInfo.getRevision() %>Hadoop version and revision
Hadoop Compiled<% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %>When Hadoop version was compiled and by whom
-
-
- - - - - - - - -<%def serverNameLink> - <%args> - ServerName serverName; - - <%java> - int infoPort = serverName.getPort() + 1; - String url = "//" + serverName.getHostname() + ":" + infoPort + "/"; - - - <%if (infoPort > 0) %> - <% serverName.getServerName() %> - <%else> - <% serverName.getServerName() %> - - diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BulkLoadObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BulkLoadObserver.java index 8e571c2403e8..a5af783360df 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BulkLoadObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BulkLoadObserver.java @@ -27,12 +27,12 @@ *
*

Exception Handling

For all functions, exception handling is done as follows: *